From c16ef04f84e4d2233e7b5daa132025c03b58467b Mon Sep 17 00:00:00 2001 From: Hayden Baker Date: Thu, 31 Aug 2023 08:50:03 -0700 Subject: [PATCH 1/5] Add flexible checksum support for non-streaming cases --- core/http-auth-aws/pom.xml | 15 + .../awssdk/http/auth/aws/AwsV4HttpSigner.java | 7 + .../internal/checksums/ConstantChecksum.java | 63 ++ .../internal/checksums/Crc32CChecksum.java | 97 +++ .../aws/internal/checksums/Crc32Checksum.java | 94 +++ .../aws/internal/checksums/Md5Checksum.java | 84 +++ .../aws/internal/checksums/SdkChecksum.java | 121 ++++ .../aws/internal/checksums/Sha1Checksum.java | 84 +++ .../internal/checksums/Sha256Checksum.java | 84 +++ .../factory/CrtBasedChecksumProvider.java | 78 +++ .../internal/checksums/factory/SdkCrc32.java | 622 +++++++++++++++++ .../internal/checksums/factory/SdkCrc32C.java | 638 ++++++++++++++++++ .../aws/internal/io/ChecksumInputStream.java | 57 ++ .../ChecksumSubscriber.java} | 49 +- .../aws/internal/io/SdkDigestInputStream.java | 109 --- .../signer/DefaultAwsV4HttpSigner.java | 25 +- .../internal/signer/DefaultChecksummer.java | 46 +- .../signer/DefaultV4PayloadSigner.java | 4 +- .../signer/DefaultV4RequestSigner.java | 13 +- .../internal/signer/FlexibleChecksummer.java | 77 +++ .../signer/PrecomputedChecksummer.java | 24 +- .../auth/aws/internal/util/ChecksumUtil.java | 116 ++++ .../http/auth/aws/signer/Checksummer.java | 58 +- .../http/auth/aws/util/SignerUtils.java | 29 +- .../internal/io/ChecksumInputStreamTest.java | 66 ++ .../io/ChecksumSubscriberTckTest.java | 43 ++ .../io/ChecksumSubscriberTest.java} | 49 +- .../signer/DefaultAwsV4HttpSignerTest.java | 35 + 28 files changed, 2580 insertions(+), 207 deletions(-) create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/ConstantChecksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Md5Checksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkChecksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha1Checksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha256Checksum.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java rename core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/{util/DigestComputingSubscriber.java => io/ChecksumSubscriber.java} (60%) delete mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/SdkDigestInputStream.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTckTest.java rename core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/{DigestComputingSubscriberTest.java => internal/io/ChecksumSubscriberTest.java} (50%) diff --git a/core/http-auth-aws/pom.xml b/core/http-auth-aws/pom.xml index 3427c81ac256..e6f7a162aaa7 100644 --- a/core/http-auth-aws/pom.xml +++ b/core/http-auth-aws/pom.xml @@ -59,6 +59,16 @@ http-auth-spi ${awsjavasdk.version} + + software.amazon.awssdk + checksums-spi + ${awsjavasdk.version} + + + software.amazon.awssdk + checksums + ${awsjavasdk.version} + org.mockito @@ -85,6 +95,11 @@ equalsverifier test + + org.reactivestreams + reactive-streams-tck + test + diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/AwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/AwsV4HttpSigner.java index d081d0a43d68..ebab354e3afe 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/AwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/AwsV4HttpSigner.java @@ -17,6 +17,7 @@ import java.time.Duration; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.auth.aws.internal.signer.DefaultAwsV4HttpSigner; import software.amazon.awssdk.http.auth.spi.HttpSigner; import software.amazon.awssdk.http.auth.spi.SignerProperty; @@ -83,6 +84,12 @@ public interface AwsV4HttpSigner extends HttpSigner { SignerProperty CHUNK_ENCODING_ENABLED = SignerProperty.create(Boolean.class, "ChunkEncodingEnabled"); + /** + * The algorithm to use for calculating a "flexible" checksum. This property is optional. + */ + SignerProperty CHECKSUM_ALGORITHM = + SignerProperty.create(ChecksumAlgorithm.class, "ChecksumAlgorithm"); + /** * Get a default implementation of a {@link AwsV4HttpSigner} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/ConstantChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/ConstantChecksum.java new file mode 100644 index 000000000000..d6e2ea41a3d7 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/ConstantChecksum.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import java.nio.charset.StandardCharsets; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Implementation of {@link SdkChecksum} to provide a constant checksum. + */ +@SdkInternalApi +public class ConstantChecksum implements SdkChecksum { + + private final String value; + + public ConstantChecksum(String value) { + this.value = value; + } + + @Override + public void update(int b) { + } + + @Override + public void update(byte[] b, int off, int len) { + } + + @Override + public long getValue() { + throw new UnsupportedOperationException("Use getChecksumBytes() instead."); + } + + @Override + public void reset() { + } + + @Override + public byte[] getChecksumBytes() { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public void mark(int readLimit) { + } + + @Override + public String getChecksum() { + return value; + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java new file mode 100644 index 000000000000..075f7dcb1044 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; +import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32C; + +/** + * Implementation of {@link SdkChecksum} to calculate an CRC32C checksum. + */ +@SdkInternalApi +public class Crc32CChecksum implements SdkChecksum { + + private Checksum crc32c; + private Checksum lastMarkedCrc32C; + private final boolean isCrtBasedChecksum; + + /** + * Creates CRT Based Crc32C checksum if Crt classpath for Crc32c is loaded, else create Sdk Implemented Crc32c + */ + public Crc32CChecksum() { + crc32c = CrtBasedChecksumProvider.createCrc32C(); + isCrtBasedChecksum = crc32c != null; + if (!isCrtBasedChecksum) { + crc32c = SdkCrc32C.create(); + } + } + + @Override + public byte[] getChecksumBytes() { + return Arrays.copyOfRange(longToByte(crc32c.getValue()), 4, 8); + } + + + @Override + public void mark(int readLimit) { + this.lastMarkedCrc32C = cloneChecksum(crc32c); + } + + @Override + public void update(int b) { + crc32c.update(b); + } + + @Override + public void update(byte[] b, int off, int len) { + crc32c.update(b, off, len); + } + + @Override + public long getValue() { + return crc32c.getValue(); + } + + @Override + public void reset() { + if (lastMarkedCrc32C == null) { + crc32c.reset(); + } else { + crc32c = cloneChecksum(lastMarkedCrc32C); + } + } + + + private Checksum cloneChecksum(Checksum checksum) { + if (isCrtBasedChecksum) { + try { + Method method = checksum.getClass().getDeclaredMethod("clone"); + return (Checksum) method.invoke(checksum); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Could not clone checksum class " + checksum.getClass(), e); + } + } else { + return (Checksum) ((SdkCrc32C) checksum).clone(); + + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java new file mode 100644 index 000000000000..d5283d8ec359 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java @@ -0,0 +1,94 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; +import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32; + +/** + * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum. + */ +@SdkInternalApi +public class Crc32Checksum implements SdkChecksum { + + private Checksum crc32; + private Checksum lastMarkedCrc32; + private final boolean isCrtBasedChecksum; + + /** + * Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32. + */ + public Crc32Checksum() { + crc32 = CrtBasedChecksumProvider.createCrc32(); + isCrtBasedChecksum = crc32 != null; + if (!isCrtBasedChecksum) { + crc32 = SdkCrc32.create(); + } + } + + @Override + public byte[] getChecksumBytes() { + return Arrays.copyOfRange(longToByte(crc32.getValue()), 4, 8); + } + + @Override + public void mark(int readLimit) { + this.lastMarkedCrc32 = cloneChecksum(crc32); + } + + @Override + public void update(int b) { + crc32.update(b); + } + + @Override + public void update(byte[] b, int off, int len) { + crc32.update(b, off, len); + } + + @Override + public long getValue() { + return crc32.getValue(); + } + + @Override + public void reset() { + if ((lastMarkedCrc32 == null)) { + crc32.reset(); + } else { + crc32 = cloneChecksum(lastMarkedCrc32); + } + } + + private Checksum cloneChecksum(Checksum checksum) { + if (isCrtBasedChecksum) { + try { + Method method = checksum.getClass().getDeclaredMethod("clone"); + return (Checksum) method.invoke(checksum); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Could not clone checksum class " + checksum.getClass(), e); + } + } else { + return (Checksum) ((SdkCrc32) checksum).clone(); + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Md5Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Md5Checksum.java new file mode 100644 index 000000000000..9c2d1a1eeffc --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Md5Checksum.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import java.security.MessageDigest; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Implementation of {@link SdkChecksum} to calculate an MD5 checksum. + */ +@SdkInternalApi +public class Md5Checksum implements SdkChecksum { + + private MessageDigest digest; + + private MessageDigest digestLastMarked; + + public Md5Checksum() { + this.digest = getDigest(); + } + + @Override + public void update(int b) { + digest.update((byte) b); + } + + @Override + public void update(byte[] b, int off, int len) { + digest.update(b, off, len); + } + + @Override + public long getValue() { + throw new UnsupportedOperationException("Use getChecksumBytes() instead."); + } + + @Override + public void reset() { + digest = (digestLastMarked == null) + // This is necessary so that should there be a reset without a + // preceding mark, the MD5 would still be computed correctly. + ? getDigest() + : cloneFrom(digestLastMarked); + } + + private MessageDigest getDigest() { + try { + return MessageDigest.getInstance("MD5"); + } catch (Exception e) { + throw new IllegalStateException("Unexpected error creating MD5 checksum", e); + } + } + + @Override + public byte[] getChecksumBytes() { + return digest.digest(); + } + + @Override + public void mark(int readLimit) { + digestLastMarked = cloneFrom(digest); + } + + private MessageDigest cloneFrom(MessageDigest from) { + try { + return (MessageDigest) from.clone(); + } catch (CloneNotSupportedException e) { // should never occur + throw new IllegalStateException("unexpected", e); + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkChecksum.java new file mode 100644 index 000000000000..717bb246f393 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkChecksum.java @@ -0,0 +1,121 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import static software.amazon.awssdk.utils.BinaryUtils.toHex; + +import java.nio.ByteBuffer; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Extension of {@link Checksum} to support checksums and checksum validations used by the SDK that + * are not provided by the JDK. + */ +@SdkInternalApi +public interface SdkChecksum extends Checksum { + + /** + * Returns the computed checksum in a byte array rather than the long provided by + * {@link #getValue()}. + * + * @return byte[] containing the checksum + */ + byte[] getChecksumBytes(); + + /** + * Allows marking a checksum for checksums that support the ability to mark and reset. + * + * @param readLimit the maximum limit of bytes that can be read before the mark position becomes invalid. + */ + void mark(int readLimit); + + /** + * Updates the current checksum with the specified array of bytes. + * + * @param b the array of bytes to update the checksum with + * + * @throws NullPointerException + * if {@code b} is {@code null} + */ + default void update(byte[] b) { + update(b, 0, b.length); + } + + + /** + * Updates the current checksum with the bytes from the specified buffer. + * + * The checksum is updated with the remaining bytes in the buffer, starting + * at the buffer's position. Upon return, the buffer's position will be + * updated to its limit; its limit will not have been changed. + * + * @apiNote For best performance with DirectByteBuffer and other ByteBuffer + * implementations without a backing array implementers of this interface + * should override this method. + * + * @implSpec The default implementation has the following behavior.
+ * For ByteBuffers backed by an accessible byte array. + * 
{@code
+     * update(buffer.array(),
+     *        buffer.position() + buffer.arrayOffset(),
+     *        buffer.remaining());
+     * }
+ * For ByteBuffers not backed by an accessible byte array. + * 
{@code
+     * byte[] b = new byte[Math.min(buffer.remaining(), 4096)];
+     * while (buffer.hasRemaining()) {
+     *     int length = Math.min(buffer.remaining(), b.length);
+     *     buffer.get(b, 0, length);
+     *     update(b, 0, length);
+     * }
+     * }
+ * + * @param buffer the ByteBuffer to update the checksum with + * + * @throws NullPointerException + * if {@code buffer} is {@code null} + * + */ + default void update(ByteBuffer buffer) { + int pos = buffer.position(); + int limit = buffer.limit(); + int rem = limit - pos; + if (rem <= 0) { + return; + } + if (buffer.hasArray()) { + update(buffer.array(), pos + buffer.arrayOffset(), rem); + } else { + byte[] b = new byte[Math.min(buffer.remaining(), 4096)]; + while (buffer.hasRemaining()) { + int length = Math.min(buffer.remaining(), b.length); + buffer.get(b, 0, length); + update(b, 0, length); + } + } + buffer.position(limit); + } + + /** + * Return an encoded-string representation of the checksum. By default, this returns a String that is the lowercase, base16 + * (hex) representation of the checksum. + * + */ + default String getChecksum() { + return toHex(getChecksumBytes()); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha1Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha1Checksum.java new file mode 100644 index 000000000000..eff87e2a5792 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha1Checksum.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import java.security.MessageDigest; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Implementation of {@link SdkChecksum} to calculate an Sha-1 checksum. + */ +@SdkInternalApi +public class Sha1Checksum implements SdkChecksum { + + private MessageDigest digest; + + private MessageDigest digestLastMarked; + + public Sha1Checksum() { + this.digest = getDigest(); + } + + @Override + public void update(int b) { + digest.update((byte) b); + } + + @Override + public void update(byte[] b, int off, int len) { + digest.update(b, off, len); + } + + @Override + public long getValue() { + throw new UnsupportedOperationException("Use getChecksumBytes() instead."); + } + + @Override + public void reset() { + digest = (digestLastMarked == null) + // This is necessary so that should there be a reset without a + // preceding mark, the Sha-1 would still be computed correctly. + ? getDigest() + : cloneFrom(digestLastMarked); + } + + private MessageDigest getDigest() { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (Exception e) { + throw new IllegalStateException("Unexpected error creating SHA-1 checksum", e); + } + } + + @Override + public byte[] getChecksumBytes() { + return digest.digest(); + } + + @Override + public void mark(int readLimit) { + digestLastMarked = cloneFrom(digest); + } + + private MessageDigest cloneFrom(MessageDigest from) { + try { + return (MessageDigest) from.clone(); + } catch (CloneNotSupportedException e) { // should never occur + throw new IllegalStateException("unexpected", e); + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha256Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha256Checksum.java new file mode 100644 index 000000000000..6b25f45d5e6c --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Sha256Checksum.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums; + +import java.security.MessageDigest; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Implementation of {@link SdkChecksum} to calculate an Sha-256 Checksum. + */ +@SdkInternalApi +public class Sha256Checksum implements SdkChecksum { + + private MessageDigest digest; + + private MessageDigest digestLastMarked; + + public Sha256Checksum() { + this.digest = getDigest(); + } + + @Override + public void update(int b) { + digest.update((byte) b); + } + + @Override + public void update(byte[] b, int off, int len) { + digest.update(b, off, len); + } + + @Override + public long getValue() { + throw new UnsupportedOperationException("Use getChecksumBytes() instead."); + } + + @Override + public void reset() { + digest = (digestLastMarked == null) + // This is necessary so that should there be a reset without a + // preceding mark, the Sha-256 would still be computed correctly. + ? getDigest() + : cloneFrom(digestLastMarked); + } + + private MessageDigest getDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (Exception e) { + throw new IllegalStateException("Unexpected error creating SHA-256 checksum", e); + } + } + + @Override + public byte[] getChecksumBytes() { + return digest.digest(); + } + + @Override + public void mark(int readLimit) { + digestLastMarked = cloneFrom(digest); + } + + private MessageDigest cloneFrom(MessageDigest from) { + try { + return (MessageDigest) from.clone(); + } catch (CloneNotSupportedException e) { // should never occur + throw new IllegalStateException("unexpected", e); + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java new file mode 100644 index 000000000000..b184f48a7b27 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; + +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.ClassLoaderHelper; +import software.amazon.awssdk.utils.Lazy; +import software.amazon.awssdk.utils.Logger; + + +/** + * Class to load the Crt based checksum from aws-crt-java library if it is present in class path. + */ +@SdkInternalApi +public final class CrtBasedChecksumProvider { + + public static final Logger LOG = Logger.loggerFor(CrtBasedChecksumProvider.class); + private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C"; + private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32"; + private static final Lazy>> CRT_CRC32_CLASS_LOADER = + new Lazy<>(() -> initializeCrtChecksumClass(CRT_CLASSPATH_FOR_CRC32)); + private static final Lazy>> CRT_CRC32_C_CLASS_LOADER = + new Lazy<>(() -> initializeCrtChecksumClass(CRT_CLASSPATH_FOR_CRC32C)); + + private CrtBasedChecksumProvider() { + } + + public static Checksum createCrc32() { + return createCrtBasedChecksum(CRT_CRC32_CLASS_LOADER); + } + + public static Checksum createCrc32C() { + return createCrtBasedChecksum(CRT_CRC32_C_CLASS_LOADER); + } + + private static Checksum createCrtBasedChecksum(Lazy>> lazyClassLoader) { + return lazyClassLoader.getValue().map( + checksumClass -> { + try { + return (Checksum) checksumClass.getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException e) { + return null; + } + }).orElse(null); + } + + private static Optional> initializeCrtChecksumClass(String classPath) { + try { + return Optional.of(ClassLoaderHelper.loadClass(classPath, false)); + } catch (ClassNotFoundException e) { + LOG.debug(() -> "Cannot find the " + classPath + " class." + + " To invoke a request that requires a CRT based checksums.", e); + return Optional.empty(); + } + } + + public static byte[] longToByte(Long input) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(input); + return buffer.array(); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java new file mode 100644 index 000000000000..db605a0373d4 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java @@ -0,0 +1,622 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/* + * THIS FILE HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. + * + * The code comes from PureJavaCrc32.java in Apache Commons Codec 1.11. + * It has been modified to add a createCopy() method. + * The createCopy method is used to save current c checksum state when the checksum is marked. + */ +@SdkInternalApi +public final class SdkCrc32 implements Checksum, Cloneable { + + /* + * CRC-32 lookup tables generated by the polynomial 0xEDB88320. + * See also TestPureJavaCrc32.Table. + */ + private static final int[] T = { + /* T8_0 */ + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D, + /* T8_1 */ + 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, + 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, + 0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, + 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF, + 0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192, + 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496, + 0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A, + 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E, + 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761, + 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265, + 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69, + 0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D, + 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530, + 0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034, + 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38, + 0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C, + 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6, + 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2, + 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE, + 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA, + 0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97, + 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93, + 0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F, + 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B, + 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864, + 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60, + 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C, + 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768, + 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35, + 0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31, + 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D, + 0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539, + 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88, + 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C, + 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180, + 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484, + 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9, + 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD, + 0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1, + 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5, + 0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A, + 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E, + 0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522, + 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026, + 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B, + 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F, + 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773, + 0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277, + 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D, + 0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189, + 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85, + 0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81, + 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC, + 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8, + 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4, + 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0, + 0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F, + 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B, + 0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27, + 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23, + 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E, + 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, + 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, + 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72, + /* T8_2 */ + 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, + 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, + 0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, + 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D, + 0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29, + 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5, + 0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91, + 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D, + 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9, + 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065, + 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901, + 0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD, + 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9, + 0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315, + 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71, + 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD, + 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399, + 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45, + 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221, + 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD, + 0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9, + 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835, + 0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151, + 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D, + 0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579, + 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5, + 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1, + 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D, + 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609, + 0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5, + 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1, + 0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D, + 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9, + 0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05, + 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461, + 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD, + 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9, + 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75, + 0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711, + 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD, + 0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339, + 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5, + 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281, + 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D, + 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049, + 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895, + 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1, + 0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D, + 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819, + 0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5, + 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1, + 0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D, + 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69, + 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5, + 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1, + 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D, + 0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9, + 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625, + 0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41, + 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D, + 0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89, + 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, + 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, + 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED, + /* T8_3 */ + 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, + 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, + 0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701, + 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056, + 0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871, + 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26, + 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E, + 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9, + 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0, + 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787, + 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F, + 0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68, + 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F, + 0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018, + 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0, + 0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7, + 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3, + 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084, + 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C, + 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B, + 0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C, + 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B, + 0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3, + 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4, + 0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED, + 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA, + 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002, + 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755, + 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72, + 0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825, + 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D, + 0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA, + 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5, + 0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82, + 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A, + 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D, + 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A, + 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D, + 0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5, + 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2, + 0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB, + 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC, + 0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04, + 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953, + 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174, + 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623, + 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B, + 0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC, + 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8, + 0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF, + 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907, + 0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50, + 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677, + 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120, + 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98, + 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF, + 0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6, + 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981, + 0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639, + 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E, + 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949, + 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, + 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, + 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1, + /* T8_4 */ + 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, + 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, + 0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111, + 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1, + 0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52, + 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92, + 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693, + 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053, + 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4, + 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314, + 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15, + 0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5, + 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256, + 0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496, + 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997, + 0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57, + 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299, + 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459, + 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958, + 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98, + 0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B, + 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB, + 0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA, + 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A, + 0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D, + 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D, + 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C, + 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C, + 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F, + 0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF, + 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE, + 0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E, + 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42, + 0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82, + 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183, + 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743, + 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0, + 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00, + 0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601, + 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1, + 0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546, + 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386, + 0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87, + 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847, + 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4, + 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404, + 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905, + 0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5, + 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B, + 0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB, + 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA, + 0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A, + 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589, + 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349, + 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48, + 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888, + 0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F, + 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF, + 0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE, + 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E, + 0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D, + 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, + 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, + 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C, + /* T8_5 */ + 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, + 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, + 0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, + 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5, + 0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035, + 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223, + 0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258, + 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E, + 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798, + 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E, + 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5, + 0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3, + 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503, + 0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715, + 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E, + 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578, + 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2, + 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4, + 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF, + 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9, + 0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59, + 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F, + 0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834, + 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22, + 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4, + 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2, + 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99, + 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F, + 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F, + 0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79, + 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02, + 0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14, + 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676, + 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460, + 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B, + 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D, + 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED, + 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB, + 0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680, + 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496, + 0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340, + 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156, + 0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D, + 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B, + 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB, + 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD, + 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6, + 0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0, + 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A, + 0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C, + 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77, + 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61, + 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81, + 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97, + 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC, + 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA, + 0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C, + 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A, + 0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41, + 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957, + 0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7, + 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, + 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, + 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC, + /* T8_6 */ + 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, + 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, + 0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA, + 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9, + 0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653, + 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240, + 0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834, + 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27, + 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301, + 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712, + 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66, + 0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975, + 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF, + 0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC, + 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8, + 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB, + 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4, + 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7, + 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183, + 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590, + 0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A, + 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739, + 0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D, + 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E, + 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678, + 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B, + 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F, + 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C, + 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6, + 0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5, + 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1, + 0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2, + 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F, + 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C, + 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08, + 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B, + 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1, + 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2, + 0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6, + 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5, + 0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3, + 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0, + 0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794, + 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387, + 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D, + 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E, + 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A, + 0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49, + 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516, + 0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105, + 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71, + 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62, + 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8, + 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB, + 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF, + 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC, + 0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A, + 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899, + 0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED, + 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE, + 0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044, + 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, + 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, + 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30, + /* T8_7 */ + 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, + 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, + 0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, + 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC, + 0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8, + 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832, + 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D, + 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387, + 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5, + 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F, + 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00, + 0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA, + 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E, + 0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64, + 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B, + 0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1, + 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E, + 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4, + 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB, + 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041, + 0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425, + 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF, + 0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90, + 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A, + 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758, + 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2, + 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED, + 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217, + 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673, + 0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889, + 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6, + 0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C, + 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239, + 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3, + 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C, + 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776, + 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312, + 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8, + 0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7, + 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D, + 0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F, + 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95, + 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA, + 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520, + 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144, + 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE, + 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1, + 0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B, + 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4, + 0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E, + 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61, + 0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B, + 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF, + 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05, + 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A, + 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0, + 0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282, + 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78, + 0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937, + 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD, + 0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9, + 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53, + 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, + 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 + }; + /** + * the current CRC value, bit-flipped + */ + private int crc; + + private SdkCrc32() { + reset(); + } + + + private SdkCrc32(int crc) { + this.crc = crc; + + } + + public static SdkCrc32 create() { + return new SdkCrc32(); + } + + @Override + public long getValue() { + return (~crc) & 0xffffffffL; + } + + @Override + public void reset() { + crc = 0xffffffff; + } + + @Override + public void update(byte[] b, int offset, int len) { + int localCrc = crc; + + int remainder = len & 0x7; + int i = offset; + for (int end = offset + len - remainder; i < end; i += 8) { + int x = localCrc ^ + ((((b[i] << 24) >>> 24) + ((b[i + 1] << 24) >>> 16)) + + (((b[i + 2] << 24) >>> 8) + (b[i + 3] << 24))); + + localCrc = ((T[((x << 24) >>> 24) + 0x700] ^ T[((x << 16) >>> 24) + 0x600]) ^ + (T[((x << 8) >>> 24) + 0x500] ^ T[(x >>> 24) + 0x400])) ^ + ((T[((b[i + 4] << 24) >>> 24) + 0x300] ^ T[((b[i + 5] << 24) >>> 24) + 0x200]) ^ + (T[((b[i + 6] << 24) >>> 24) + 0x100] ^ T[((b[i + 7] << 24) >>> 24)])); + } + + /* loop unroll - duff's device style */ + + for (int index = 0; index < remainder; index++) { + localCrc = (localCrc >>> 8) ^ T[((localCrc ^ b[i++]) << 24) >>> 24]; + } + + // Publish crc out to object + crc = localCrc; + } + + @Override + public void update(int b) { + crc = (crc >>> 8) ^ T[(((crc ^ b) << 24) >>> 24)]; + } + + @Override + public Object clone() { + return new SdkCrc32(crc); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java new file mode 100644 index 000000000000..a408602ba1ae --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java @@ -0,0 +1,638 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + + +/* + * THIS FILE HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. + * + * The code comes from PureJavaCrc32C.java in Apache Commons Codec 1.11. + * It has been modified to add a createCopy() method. + * The createCopy method is used to save current checksum state when the checksum is marked. + */ +@SdkInternalApi +public final class SdkCrc32C implements Checksum, Cloneable { + + private static final int T8_0_START = 0 * 256; + private static final int T8_1_START = 1 * 256; + private static final int T8_2_START = 2 * 256; + private static final int T8_3_START = 3 * 256; + private static final int T_8_4_START = 4 * 256; + private static final int T8_5_START = 5 * 256; + private static final int T8_6_START = 6 * 256; + private static final int T8_7_START = 7 * 256; + + // CRC polynomial tables generated by: + // java -cp build/test/classes/:build/classes/ \ + // org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78 + private static final int[] T = { + /* T8_0 */ + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, + /* T8_1 */ + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, + 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, + 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, + 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, + 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, + 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, + 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, + 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, + 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, + 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, + 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, + 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, + 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, + 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, + 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, + 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, + 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, + 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, + 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, + 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, + 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, + 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, + 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, + 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, + 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, + 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, + 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, + 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, + 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, + 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, + 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, + 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, + 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, + 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, + 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, + 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, + 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, + 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, + 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, + 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, + 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, + 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, + 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, + 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, + 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, + 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, + 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, + 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, + 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, + 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, + /* T8_2 */ + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, + 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, + 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, + 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, + 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, + 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, + 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, + 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, + 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, + 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, + 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, + 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, + 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, + 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, + 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, + 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, + 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, + 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, + 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, + 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, + 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, + 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, + 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, + 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, + 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, + 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, + 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, + 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, + 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, + 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, + 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, + 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, + 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, + 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, + 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, + 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, + 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, + 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, + 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, + 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, + 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, + 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, + 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, + 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, + 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, + 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, + 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, + 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, + 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, + 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, + /* T8_3 */ + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, + 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, + 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, + 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, + 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, + 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, + 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, + 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, + 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, + 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, + 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, + 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, + 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, + 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, + 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, + 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, + 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, + 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, + 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, + 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, + 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, + 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, + 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, + 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, + 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, + 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, + 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, + 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, + 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, + 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, + 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, + 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, + 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, + 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, + 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, + 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, + 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, + 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, + 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, + 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, + 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, + 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, + 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, + 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, + 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, + 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, + 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, + 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, + 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, + 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, + /* T8_4 */ + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, + 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, + 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, + 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, + 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, + 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, + 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, + 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, + 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, + 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, + 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, + 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, + 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, + 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, + 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, + 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, + 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, + 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, + 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, + 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, + 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, + 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, + 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, + 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, + 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, + 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, + 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, + 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, + 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, + 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, + 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, + 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, + 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, + 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, + 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, + 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, + 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, + 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, + 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, + 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, + 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, + 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, + 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, + 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, + 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, + 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, + 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, + 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, + 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, + 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, + /* T8_5 */ + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, + 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, + 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, + 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, + 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, + 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, + 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, + 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, + 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, + 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, + 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, + 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, + 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, + 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, + 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, + 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, + 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, + 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, + 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, + 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, + 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, + 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, + 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, + 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, + 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, + 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, + 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, + 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, + 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, + 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, + 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, + 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, + 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, + 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, + 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, + 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, + 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, + 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, + 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, + 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, + 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, + 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, + 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, + 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, + 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, + 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, + 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, + 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, + 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, + 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, + /* T8_6 */ + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, + 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, + 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, + 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, + 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, + 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, + 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, + 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, + 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, + 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, + 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, + 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, + 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, + 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, + 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, + 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, + 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, + 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, + 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, + 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, + 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, + 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, + 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, + 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, + 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, + 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, + 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, + 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, + 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, + 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, + 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, + 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, + 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, + 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, + 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, + 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, + 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, + 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, + 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, + 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, + 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, + 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, + 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, + 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, + 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, + 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, + 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, + 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, + 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, + 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, + /* T8_7 */ + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, + 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, + 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, + 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, + 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, + 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, + 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, + 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, + 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, + 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, + 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, + 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, + 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, + 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, + 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, + 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, + 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, + 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, + 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, + 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, + 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, + 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, + 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, + 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, + 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, + 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, + 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, + 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, + 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, + 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, + 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, + 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, + 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, + 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, + 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, + 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, + 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, + 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, + 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, + 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, + 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, + 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, + 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, + 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, + 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, + 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, + 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, + 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, + 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, + 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 + }; + /** + * the current CRC value, bit-flipped + */ + private int crc; + + private SdkCrc32C() { + reset(); + } + + private SdkCrc32C(int crc) { + this.crc = crc; + } + + public static SdkCrc32C create() { + return new SdkCrc32C(); + } + + @Override + public long getValue() { + long ret = crc; + return (~ret) & 0xffffffffL; + } + + @Override + public void reset() { + crc = 0xffffffff; + } + + @Override + public void update(byte[] b, int off, int len) { + int localCrc = crc; + + while (len > 7) { + int c0 = (b[off + 0] ^ localCrc) & 0xff; + localCrc >>>= 8; + int c1 = (b[off + 1] ^ (localCrc)) & 0xff; + localCrc >>>= 8; + int c2 = (b[off + 2] ^ (localCrc)) & 0xff; + localCrc >>>= 8; + int c3 = (b[off + 3] ^ (localCrc)) & 0xff; + localCrc = (T[T8_7_START + c0] ^ T[T8_6_START + c1]) ^ + (T[T8_5_START + c2] ^ T[T_8_4_START + c3]); + int c4 = b[off + 4] & 0xff; + int c5 = b[off + 5] & 0xff; + int c6 = b[off + 6] & 0xff; + int c7 = b[off + 7] & 0xff; + + localCrc ^= (T[T8_3_START + c4] ^ T[T8_2_START + c5]) ^ + (T[T8_1_START + c6] ^ T[T8_0_START + c7]); + + off += 8; + len -= 8; + } + + /* loop unroll - duff's device style */ + for (int index = 0; index < len; index++) { + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + } + + // Publish crc out to object + crc = localCrc; + } + + @Override + public void update(int b) { + crc = (crc >>> 8) ^ T[T8_0_START + ((crc ^ b) & 0xff)]; + } + + @Override + public Object clone() { + return new SdkCrc32C(crc); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java new file mode 100644 index 000000000000..625af50d4b56 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.io; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + +@SdkInternalApi +public class ChecksumInputStream extends FilterInputStream { + + private final Collection checksums = new ArrayList<>(); + + public ChecksumInputStream(InputStream stream, Collection checksums) { + super(stream); + this.checksums.addAll(checksums); + } + + @Override + public int read() throws IOException { + byte[] b = new byte[1]; + int read = read(b, 0, 1); + if (read > 0) { + checksums.forEach(checksum -> checksum.update(b, 0, 1)); + } + + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int read = in.read(b, off, len); + if (read > 0) { + checksums.forEach(checksum -> checksum.update(b, off, read)); + } + + return read; + } +} + diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/DigestComputingSubscriber.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java similarity index 60% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/DigestComputingSubscriber.java rename to core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java index a1ef9b1c027a..072abe63a202 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/DigestComputingSubscriber.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java @@ -13,28 +13,29 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.util; +package software.amazon.awssdk.http.auth.aws.internal.io; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collection; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.zip.Checksum; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; @SdkInternalApi -public final class DigestComputingSubscriber implements Subscriber { - private final CompletableFuture digestBytes = new CompletableFuture<>(); - private final MessageDigest messageDigest; +public final class ChecksumSubscriber implements Subscriber { + private final CompletableFuture checksumming = new CompletableFuture<>(); + private final Collection checksums = new ArrayList<>(); private volatile boolean canceled = false; private volatile Subscription subscription; - public DigestComputingSubscriber(MessageDigest messageDigest) { - this.messageDigest = messageDigest; + public ChecksumSubscriber(Collection consumers) { + this.checksums.addAll(consumers); - digestBytes.whenComplete((r, t) -> { + checksumming.whenComplete((r, t) -> { if (t instanceof CancellationException) { synchronized (this) { canceled = true; @@ -46,18 +47,10 @@ public DigestComputingSubscriber(MessageDigest messageDigest) { }); } - public static DigestComputingSubscriber forSha256() { - try { - return new DigestComputingSubscriber(MessageDigest.getInstance("SHA-256")); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Unable to create SHA-256 computing subscriber" + e.getMessage()); - } - } - @Override public void onSubscribe(Subscription subscription) { synchronized (this) { - if (!canceled) { + if (!canceled && this.subscription == null) { this.subscription = subscription; subscription.request(Long.MAX_VALUE); } else { @@ -69,21 +62,31 @@ public void onSubscribe(Subscription subscription) { @Override public void onNext(ByteBuffer byteBuffer) { if (!canceled) { - messageDigest.update(byteBuffer); + byte[] buf; + + if (byteBuffer.hasArray()) { + buf = byteBuffer.array(); + } else { + buf = new byte[byteBuffer.remaining()]; + byteBuffer.get(buf); + } + + // We have to use a byte[], since update() is java 9+ + checksums.forEach(checksum -> checksum.update(buf, 0, buf.length)); } } @Override public void onError(Throwable throwable) { - digestBytes.completeExceptionally(throwable); + checksumming.completeExceptionally(throwable); } @Override public void onComplete() { - digestBytes.complete(messageDigest.digest()); + checksumming.complete(null); } - public CompletableFuture digestBytes() { - return digestBytes; + public CompletableFuture checksum() { + return checksumming; } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/SdkDigestInputStream.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/SdkDigestInputStream.java deleted file mode 100644 index 64e42deea0bd..000000000000 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/SdkDigestInputStream.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.io; - -import java.io.IOException; -import java.io.InputStream; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.utils.IoUtils; - -/** - * Base class for AWS Java SDK specific {@link DigestInputStream}. - */ -@SdkInternalApi -public final class SdkDigestInputStream extends DigestInputStream implements Releasable { - private static final int SKIP_BUF_SIZE = 2 * 1024; - - public SdkDigestInputStream(InputStream stream, MessageDigest digest) { - super(stream, digest); - } - - // https://github.com/aws/aws-sdk-java/issues/232 - - /** - * Skips over and discards {@code n} bytes of data from this input stream, while taking the skipped bytes into account - * for digest calculation. The {@code skip} method may, for a variety of reasons, end up skipping over some smaller - * number of bytes, possibly - * {@code 0}. This may result from any of a number of conditions; - * reaching end of file before {@code n} bytes have been skipped is only one possibility. The actual number of bytes - * skipped is returned. If - * {@code n} is negative, no bytes are skipped. - * - *

- * The {@code skip} method of this class creates a byte array and then repeatedly reads into it until n - * bytes have been read or the end of the stream has been reached. Subclasses are encouraged to provide a more efficient - * implementation of this method. For instance, the implementation may depend on the ability to seek. - * - * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. - * @throws IOException if the stream does not support seek, or if some other I/O error occurs. - */ - @Override - public long skip(long n) throws IOException { - if (n <= 0) { - return n; - } - byte[] b = new byte[(int) Math.min(SKIP_BUF_SIZE, n)]; - long m = n; // remaining number of bytes to read - while (m > 0) { - int len = read(b, 0, (int) Math.min(m, b.length)); - if (len == -1) { - return n - m; - } - m -= len; - } - assert m == 0; - return n; - } - - @Override - public void release() { - // Don't call IOUtils.release(in, null) or else could lead to infinite loop - IoUtils.closeQuietly(this, null); - if (in instanceof Releasable) { - // This allows any underlying stream that has the close operation - // disabled to be truly released - Releasable r = (Releasable) in; - r.release(); - } - } - - /** - * {@inheritdoc} - */ - @Override - public int read() throws IOException { - int ch = in.read(); - if (ch != -1) { - digest.update((byte) ch); - } - return ch; - } - - /** - * {@inheritdoc} - */ - @Override - public int read(byte[] b, int off, int len) throws IOException { - int result = in.read(b, off, len); - if (result != -1) { - digest.update(b, off, result); - } - return result; - } -} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index 70800a4e5623..6b155b52e7ff 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -22,7 +22,6 @@ import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.STREAMING_SIGNED_PAYLOAD_TRAILER; import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER; import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.UNSIGNED_PAYLOAD; -import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_TRAILER; import java.time.Clock; @@ -112,7 +111,7 @@ private static Checksummer checksummer(SignRequest STREAMING_EVENTS_PAYLOAD); @@ -125,7 +124,7 @@ private static Checksummer checksummer(SignRequest STREAMING_SIGNED_PAYLOAD); } - return Checksummer.create(); + return Checksummer.create(request.property(CHECKSUM_ALGORITHM)); } if (isChunkEncoding) { @@ -135,7 +134,7 @@ private static Checksummer checksummer(SignRequest UNSIGNED_PAYLOAD); + return Checksummer.create(UNSIGNED_PAYLOAD, request.property(CHECKSUM_ALGORITHM)); } private static V4PayloadSigner v4PayloadSigner( @@ -177,8 +176,7 @@ private static SyncSignedRequest doSign(SyncSignRequest doSign(AsyncSignRequest futureV4Context = - checksummer.checksum(request.payload().orElse(null)).thenApply( - checksum -> { - requestBuilder.putHeader(X_AMZ_CONTENT_SHA256, checksum); - return requestSigner.sign(requestBuilder); - }); + checksummer.checksum(request.payload().orElse(null), requestBuilder) + .thenApply(__ -> requestSigner.sign(requestBuilder)); return futureV4Context.thenApply( v4Context -> AsyncSignedRequest.builder() @@ -235,10 +230,10 @@ private static boolean isEventStreaming(SdkHttpRequest request) { return "application/vnd.amazon.eventstream".equals(request.firstMatchingHeader(Header.CONTENT_TYPE).orElse("")); } - private static boolean hasTrailer(SdkHttpRequest request) { - // TODO(sra-identity-and-auth): Trailer would be determined by being a flexible-checksum enabled request, we will need - // to update this once flexible checksums is enabled - return request.firstMatchingHeader(X_AMZ_TRAILER).isPresent(); + private static boolean hasTrailer(SignRequest request) { + // Flexible checksums dictates adding a trailer when signing, but there may be existing trailers on + // the request (in the future) + return request.hasProperty(CHECKSUM_ALGORITHM) || request.request().firstMatchingHeader(X_AMZ_TRAILER).isPresent(); } /** diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java index e840398fdc9c..1760024ff066 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java @@ -15,39 +15,59 @@ package software.amazon.awssdk.http.auth.aws.internal.signer; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.fromChecksumAlgorithm; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.readAll; +import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.getBinaryRequestPayloadStream; -import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.hash; import static software.amazon.awssdk.utils.BinaryUtils.toHex; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.concurrent.CompletableFuture; import org.reactivestreams.Publisher; -import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.http.ContentStreamProvider; -import software.amazon.awssdk.http.auth.aws.internal.util.DigestComputingSubscriber; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; +import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumInputStream; +import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumSubscriber; import software.amazon.awssdk.http.auth.aws.signer.Checksummer; -import software.amazon.awssdk.utils.BinaryUtils; -@SdkProtectedApi +/** + * The default implementation of a checksummer. By default, this will calculate the SHA256 checksum of a payload and add it as the + * value for the 'x-amz-content-sha256' header on the request. + */ +@SdkInternalApi public final class DefaultChecksummer implements Checksummer { @Override - public String checksum(ContentStreamProvider payload) { - InputStream payloadStream = getBinaryRequestPayloadStream(payload); - return toHex( - hash(payloadStream) + public void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder request) { + SdkChecksum sdkChecksum = fromChecksumAlgorithm(SHA256); + InputStream payloadStream = new ChecksumInputStream( + getBinaryRequestPayloadStream(payload), + Collections.singletonList(sdkChecksum) ); + + readAll(payloadStream); + + request.putHeader(X_AMZ_CONTENT_SHA256, toHex(sdkChecksum.getChecksumBytes())); } @Override - public CompletableFuture checksum(Publisher payload) { - DigestComputingSubscriber bodyDigester = DigestComputingSubscriber.forSha256(); + public CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request) { + SdkChecksum sdkChecksum = fromChecksumAlgorithm(SHA256); + + ChecksumSubscriber checksumSubscriber = new ChecksumSubscriber(Collections.singletonList(sdkChecksum)); if (payload != null) { - payload.subscribe(bodyDigester); + payload.subscribe(checksumSubscriber); } - return bodyDigester.digestBytes().thenApply(BinaryUtils::toHex); + return checksumSubscriber.checksum().thenRun(() -> { + String checksum = toHex(sdkChecksum.getChecksumBytes()); + request.putHeader(X_AMZ_CONTENT_SHA256, checksum); + }); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4PayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4PayloadSigner.java index d5af94252a22..e47b490c269a 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4PayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4PayloadSigner.java @@ -17,7 +17,7 @@ import java.nio.ByteBuffer; import org.reactivestreams.Publisher; -import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.auth.aws.signer.V4Context; import software.amazon.awssdk.http.auth.aws.signer.V4PayloadSigner; @@ -25,7 +25,7 @@ /** * A default implementation of a payload signer that is a no-op, since payloads are most commonly unsigned. */ -@SdkProtectedApi +@SdkInternalApi public class DefaultV4PayloadSigner implements V4PayloadSigner { @Override public ContentStreamProvider sign(ContentStreamProvider payload, V4Context v4Context) { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java index 4141eb9ead27..d4763301855a 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.http.auth.aws.internal.signer; import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.AWS4_SIGNING_ALGORITHM; +import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.addHostHeader; import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.deriveSigningKey; import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.hashCanonicalRequest; @@ -45,11 +46,11 @@ public DefaultV4RequestSigner(V4Properties properties) { @Override public V4Context sign(SdkHttpRequest.Builder requestBuilder) { // Step 0: Pre-requisites - String checksum = getChecksum(requestBuilder); + String contentHash = getContentHash(requestBuilder); addHostHeader(requestBuilder); // Step 1: Create a canonical request - V4CanonicalRequest canonicalRequest = createCanonicalRequest(requestBuilder.build(), checksum); + V4CanonicalRequest canonicalRequest = createCanonicalRequest(requestBuilder.build(), contentHash); // Step 2: Create a hash of the canonical request String canonicalRequestHash = hashCanonicalRequest(canonicalRequest.getCanonicalRequestString()); @@ -63,12 +64,12 @@ public V4Context sign(SdkHttpRequest.Builder requestBuilder) { String signature = createSignature(stringToSign, signingKey); // Step 5: Return the signature to be added to the request - return new V4Context(checksum, signingKey, signature, canonicalRequest, requestBuilder); + return new V4Context(contentHash, signingKey, signature, canonicalRequest, requestBuilder); } - private String getChecksum(SdkHttpRequest.Builder requestBuilder) { - return requestBuilder.firstMatchingHeader("x-amz-content-sha256").orElseThrow( - () -> new IllegalArgumentException("Checksum must be present in the 'x-amz-content-sha256' header!") + private String getContentHash(SdkHttpRequest.Builder requestBuilder) { + return requestBuilder.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow( + () -> new IllegalArgumentException("Content hash must be present in the '" + X_AMZ_CONTENT_SHA256 + "' header!") ); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java new file mode 100644 index 000000000000..a7b2f56ea636 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.signer; + +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.fromChecksumAlgorithm; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.readAll; +import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.getBinaryRequestPayloadStream; + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; +import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumInputStream; +import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumSubscriber; +import software.amazon.awssdk.http.auth.aws.signer.Checksummer; + +/** + * A "flexible" implementation of a checksummer. + */ +@SdkInternalApi +public final class FlexibleChecksummer implements Checksummer { + private final Map checksumMap; + + public FlexibleChecksummer(Map algorithmMap) { + this.checksumMap = algorithmMap + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, v -> fromChecksumAlgorithm(v.getValue()))); + } + + @Override + public void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder request) { + InputStream payloadStream = getBinaryRequestPayloadStream(payload); + + ChecksumInputStream computingStream = new ChecksumInputStream( + payloadStream, + checksumMap.values() + ); + + readAll(computingStream); + + checksumMap.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())); + } + + @Override + public CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request) { + ChecksumSubscriber checksumSubscriber = new ChecksumSubscriber(checksumMap.values()); + + if (payload != null) { + payload.subscribe(checksumSubscriber); + } + + return checksumSubscriber.checksum().thenRun( + () -> checksumMap.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())) + ); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java index 4fc148d29605..e9adb3459f86 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java @@ -15,15 +15,23 @@ package software.amazon.awssdk.http.auth.aws.internal.signer; +import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; + import java.nio.ByteBuffer; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import org.reactivestreams.Publisher; -import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.signer.Checksummer; -@SdkProtectedApi +/** + * An implementation of a checksummer that simply passes along a computed value as a checksum. Specifically, this is used in + * the cases where the checksum is a pre-defined value that dictates specific behavior by the signer (such as aws-chunked + * payload signing, unsigned streaming with trailers, etc.). + */ +@SdkInternalApi public final class PrecomputedChecksummer implements Checksummer { private final Callable computation; @@ -33,18 +41,22 @@ public PrecomputedChecksummer(Callable computation) { } @Override - public String checksum(ContentStreamProvider payload) { + public void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder request) { try { - return computation.call(); + String checksum = computation.call(); + request.putHeader(X_AMZ_CONTENT_SHA256, checksum); } catch (Exception e) { throw new RuntimeException("Could not retrieve checksum: ", e); } } @Override - public CompletableFuture checksum(Publisher payload) { + public CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request) { try { - return CompletableFuture.completedFuture(computation.call()); + String checksum = computation.call(); + request.putHeader(X_AMZ_CONTENT_SHA256, checksum); + + return CompletableFuture.completedFuture(null); } catch (Exception e) { throw new RuntimeException("Could not retrieve checksum: ", e); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java new file mode 100644 index 000000000000..c6e542b22f68 --- /dev/null +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.util; + +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; + +import java.io.InputStream; +import java.util.Locale; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.http.auth.aws.internal.checksums.ConstantChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32CChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha1Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; + +@SdkInternalApi +public final class ChecksumUtil { + + private static final String CONSTANT_CHECKSUM = "CONSTANT"; + + private ChecksumUtil() { + } + + /** + * Get the correct checksum header name based on the checksum-algorithm. This is required to be of the form + * {@code x-amz-checksum-*}, where '*' is alphanumeric checksum-algorithm-id in lower-case form. Examples include: + *

+ * x-amz-checksum-sha256, x-amz-checksum-sha1, x-amz-checksum-crc32, x-amz-checksum-crc32c, x-amz-checksum-md5 + *

+ */ + public static String checksumHeaderName(ChecksumAlgorithm checksumAlgorithm) { + return "x-amz-checksum-" + checksumAlgorithm.algorithmId().toLowerCase(Locale.US); + } + + /** + * Gets the SdkChecksum object based on the given ChecksumAlgorithm. Instances for CRC32C, CRC32 Algorithm will be added from + * CRT Java library once they are available in release. + * + * @param checksumAlgorithm Algorithm for calculating the checksum + * @return an SdkChecksum instance. + */ + public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { + if (SHA256.algorithmId().equals(checksumAlgorithm.algorithmId())) { + return new Sha256Checksum(); + } + + if (SHA1.algorithmId().equals(checksumAlgorithm.algorithmId())) { + return new Sha1Checksum(); + } + + if (CRC32.algorithmId().equals(checksumAlgorithm.algorithmId())) { + return new Crc32Checksum(); + } + + if (CRC32C.algorithmId().equals(checksumAlgorithm.algorithmId())) { + return new Crc32CChecksum(); + } + + if (CONSTANT_CHECKSUM.equals(checksumAlgorithm.algorithmId())) { + return new ConstantChecksum(((ConstantChecksumAlgorithm) checksumAlgorithm).value); + } + + throw new UnsupportedOperationException("Checksum not supported for " + checksumAlgorithm.algorithmId()); + } + + /** + * Read the entirety of an input-stream - this is useful when the stream has side-effects (such as calculating a checksum) + * when it gets read. + */ + public static void readAll(InputStream inputStream) { + try { + byte[] buffer = new byte[4096]; + while (inputStream.read(buffer) > -1) { + } + } catch (Exception e) { + throw new RuntimeException("Could not finish reading stream: ", e); + } + } + + /** + * An implementation of a {@link ChecksumAlgorithm} that will map to {@link ConstantChecksum}, which provides a constant + * checksum. This isn't super useful, but is needed in cases such as signing, where the content-hash (a + * cryptographically-secure "checksum") can be a set of pre-defined values. + */ + public static class ConstantChecksumAlgorithm implements ChecksumAlgorithm { + + private final String value; + + public ConstantChecksumAlgorithm(String value) { + this.value = value; + } + + @Override + public String algorithmId() { + return CONSTANT_CHECKSUM; + } + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java index d2fbb82bccfe..c7baa06176bc 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java @@ -15,15 +15,28 @@ package software.amazon.awssdk.http.auth.aws.signer; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.ConstantChecksumAlgorithm; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.checksumHeaderName; +import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; + import java.nio.ByteBuffer; +import java.util.Map; import java.util.concurrent.CompletableFuture; import org.reactivestreams.Publisher; import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.internal.signer.DefaultChecksummer; +import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; +import software.amazon.awssdk.http.auth.aws.internal.signer.PrecomputedChecksummer; +import software.amazon.awssdk.utils.ImmutableMap; /** * An interface for defining how a checksum is formed from a payload synchronously and asynchronously. + *

+ * The implementation may choose to also manipulate the request with the checksum, such as adding it as a header. */ @SdkProtectedApi public interface Checksummer { @@ -35,12 +48,49 @@ static Checksummer create() { } /** - * Given a payload, calculate a checksum and return it as a string. + * Get a flexible checksummer that uses the given checksum-algorithm and the default. + */ + static Checksummer create(ChecksumAlgorithm checksumAlgorithm) { + if (checksumAlgorithm != null) { + Map checksums = ImmutableMap.of( + X_AMZ_CONTENT_SHA256, SHA256, + checksumHeaderName(checksumAlgorithm), checksumAlgorithm + ); + + return new FlexibleChecksummer(checksums); + } + return create(); + } + + /** + * Get a flexible checksummer that uses the given checksum-algorithm and the default. + */ + static Checksummer create(String checksum) { + return new PrecomputedChecksummer(() -> checksum); + } + + /** + * Get a flexible checksummer that uses the given checksum-algorithm and the default. + */ + static Checksummer create(String checksum, ChecksumAlgorithm checksumAlgorithm) { + if (checksumAlgorithm != null) { + Map checksums = ImmutableMap.of( + X_AMZ_CONTENT_SHA256, new ConstantChecksumAlgorithm(checksum), + checksumHeaderName(checksumAlgorithm), checksumAlgorithm + ); + + return new FlexibleChecksummer(checksums); + } + return create(checksum); + } + + /** + * Given a payload, calculate a checksum and add it to the request. */ - String checksum(ContentStreamProvider payload); + void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder request); /** - * Given a payload, asynchronously calculate a checksum and return a future containing it as a string. + * Given a payload, asynchronously calculate a checksum and promise to add it to the request. */ - CompletableFuture checksum(Publisher payload); + CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/util/SignerUtils.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/util/SignerUtils.java index 3a05b758dd7f..782ce123d8fd 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/util/SignerUtils.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/util/SignerUtils.java @@ -18,7 +18,6 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; @@ -29,7 +28,6 @@ import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.http.auth.aws.internal.io.SdkDigestInputStream; import software.amazon.awssdk.http.auth.aws.internal.util.FifoCache; import software.amazon.awssdk.http.auth.aws.internal.util.SignerKey; import software.amazon.awssdk.http.auth.aws.internal.util.SigningAlgorithm; @@ -190,7 +188,7 @@ private static MessageDigest getMessageDigestInstance() { try { return MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Unable to get SHA256 Function: " + e.getMessage()); + throw new RuntimeException("Unable to get SHA256 Function: ", e); } } @@ -201,7 +199,7 @@ public static InputStream getBinaryRequestPayloadStream(ContentStreamProvider st } return streamProvider.newStream(); } catch (Exception e) { - throw new RuntimeException("Unable to read request payload to sign request: " + e.getMessage()); + throw new RuntimeException("Unable to read request payload to sign request: ", e); } } @@ -210,13 +208,15 @@ public static byte[] hash(InputStream input) { // TODO(sra-identity-and-auth): Performance testing to verify if we should cache message digest instances // (thread-local) MessageDigest md = getMessageDigestInstance(); - DigestInputStream digestInputStream = new SdkDigestInputStream(input, md); - byte[] buffer = new byte[1024]; - while (digestInputStream.read(buffer) > -1) { + byte[] buf = new byte[4096]; + int read = 0; + while (read >= 0) { + read = input.read(buf); + md.update(buf, 0, read); } - return digestInputStream.getMessageDigest().digest(); + return md.digest(); } catch (Exception e) { - throw new RuntimeException("Unable to compute hash while signing request: " + e.getMessage()); + throw new RuntimeException("Unable to compute hash while signing request: ", e); } } @@ -226,19 +226,12 @@ public static byte[] hash(byte[] data) { md.update(data); return md.digest(); } catch (Exception e) { - throw new RuntimeException("Unable to compute hash while signing request: " + e.getMessage()); + throw new RuntimeException("Unable to compute hash while signing request: ", e); } } public static byte[] hash(String text) { - try { - MessageDigest md = getMessageDigestInstance(); - md.update(text.getBytes(StandardCharsets.UTF_8)); - return md.digest(); - } catch (Exception e) { - throw new RuntimeException("Unable to compute hash while signing request: " + e.getMessage()); - } + return hash(text.getBytes(StandardCharsets.UTF_8)); } - } diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java new file mode 100644 index 000000000000..a848bf6fa944 --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.io; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.readAll; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; +import software.amazon.awssdk.utils.BinaryUtils; + +public class ChecksumInputStreamTest { + + @Test + public void test_computesCorrectSha256() { + String testString = "AWS SDK for Java"; + String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; + + ByteArrayInputStream backingStream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); + SdkChecksum checksum = new Sha256Checksum(); + ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Collections.singleton(checksum)); + + readAll(inputStream); + String computedDigest = BinaryUtils.toHex(checksum.getChecksumBytes()); + + assertThat(computedDigest).isEqualTo(expectedDigest); + } + + @Test + public void test_withMultipleChecksums_shouldComputeCorrectChecksums() { + String testString = "AWS SDK for Java"; + String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; + String expectedCrc32Digest = "4ac37ece"; + + ByteArrayInputStream backingStream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); + SdkChecksum sha256Checksum = new Sha256Checksum(); + SdkChecksum crc32Checksum = new Crc32Checksum(); + ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Arrays.asList(sha256Checksum, crc32Checksum)); + + readAll(inputStream); + String computedSha256Digest = BinaryUtils.toHex(sha256Checksum.getChecksumBytes()); + String computedCrc32Digest = BinaryUtils.toHex(crc32Checksum.getChecksumBytes()); + + assertThat(computedSha256Digest).isEqualTo(expectedSha256Digest); + assertThat(computedCrc32Digest).isEqualTo(expectedCrc32Digest); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTckTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTckTest.java new file mode 100644 index 000000000000..77655d91dede --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTckTest.java @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.io; + +import java.nio.ByteBuffer; +import java.util.Collections; +import org.reactivestreams.Subscriber; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; + +/** + * TCK verifiation test for {@link ChecksumSubscriber}. + */ +public class ChecksumSubscriberTckTest extends org.reactivestreams.tck.SubscriberBlackboxVerification { + + public ChecksumSubscriberTckTest() { + super(new TestEnvironment()); + } + + @Override + public Subscriber createSubscriber() { + return new ChecksumSubscriber(Collections.singleton(new Sha256Checksum())); + } + + @Override + public ByteBuffer createElement(int i) { + return ByteBuffer.wrap(String.valueOf(i).getBytes()); + } + +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/DigestComputingSubscriberTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java similarity index 50% rename from core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/DigestComputingSubscriberTest.java rename to core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java index 1b16eb04b6e7..4be08fadbb4d 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/DigestComputingSubscriberTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws; +package software.amazon.awssdk.http.auth.aws.internal.io; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -21,39 +21,66 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static software.amazon.awssdk.utils.CompletableFutureUtils.joinLikeSync; import io.reactivex.Flowable; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import org.junit.jupiter.api.Test; import org.reactivestreams.Subscription; -import software.amazon.awssdk.http.auth.aws.internal.util.DigestComputingSubscriber; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; import software.amazon.awssdk.utils.BinaryUtils; -public class DigestComputingSubscriberTest { +public class ChecksumSubscriberTest { @Test public void test_computesCorrectSha256() { String testString = "AWS SDK for Java"; String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; - DigestComputingSubscriber subscriber = DigestComputingSubscriber.forSha256(); - + SdkChecksum checksum = new Sha256Checksum(); + ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.singleton(checksum)); Flowable publisher = Flowable.just(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); - publisher.subscribe(subscriber); - String computedDigest = BinaryUtils.toHex(subscriber.digestBytes().join()); + joinLikeSync(subscriber.checksum()); + String computedDigest = BinaryUtils.toHex(checksum.getChecksumBytes()); assertThat(computedDigest).isEqualTo(expectedDigest); } + @Test + public void test_withMultipleChecksums_shouldComputeCorrectChecksums() { + String testString = "AWS SDK for Java"; + String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; + String expectedCrc32Digest = "4ac37ece"; + + SdkChecksum sha256Checksum = new Sha256Checksum(); + SdkChecksum crc32Checksum = new Crc32Checksum(); + ChecksumSubscriber subscriber = new ChecksumSubscriber(Arrays.asList(sha256Checksum, crc32Checksum)); + Flowable publisher = Flowable.just(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); + publisher.subscribe(subscriber); + + joinLikeSync(subscriber.checksum()); + String computedSha256Digest = BinaryUtils.toHex(sha256Checksum.getChecksumBytes()); + String computedCrc32Digest = BinaryUtils.toHex(crc32Checksum.getChecksumBytes()); + + assertThat(computedSha256Digest).isEqualTo(expectedSha256Digest); + assertThat(computedCrc32Digest).isEqualTo(expectedCrc32Digest); + } + @Test public void test_futureCancelledBeforeSubscribe_cancelsSubscription() { Subscription mockSubscription = mock(Subscription.class); - DigestComputingSubscriber subscriber = DigestComputingSubscriber.forSha256(); - subscriber.digestBytes().cancel(true); + ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.emptyList()); + + subscriber.checksum().cancel(true); subscriber.onSubscribe(mockSubscription); @@ -65,12 +92,12 @@ public void test_futureCancelledBeforeSubscribe_cancelsSubscription() { public void test_publisherCallsOnError_errorPropagatedToFuture() { Subscription mockSubscription = mock(Subscription.class); - DigestComputingSubscriber subscriber = DigestComputingSubscriber.forSha256(); + ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.emptyList()); subscriber.onSubscribe(mockSubscription); RuntimeException error = new RuntimeException("error"); subscriber.onError(error); - assertThatThrownBy(subscriber.digestBytes()::join).hasCause(error); + assertThatThrownBy(subscriber.checksum()::join).hasCause(error); } } diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java index 9e774a286fac..a0486c504cdd 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java @@ -17,7 +17,9 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.http.auth.aws.AwsV4HttpSigner.AUTH_LOCATION; +import static software.amazon.awssdk.http.auth.aws.AwsV4HttpSigner.CHECKSUM_ALGORITHM; import static software.amazon.awssdk.http.auth.aws.AwsV4HttpSigner.CHUNK_ENCODING_ENABLED; import static software.amazon.awssdk.http.auth.aws.AwsV4HttpSigner.EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.AwsV4HttpSigner.PAYLOAD_SIGNING_ENABLED; @@ -231,4 +233,37 @@ public void sign_WithPayloadSigningFalseAndChunkEncodingTrueWithoutTrailer_Throw assertThrows(UnsupportedOperationException.class, () -> signer.sign(request)); } + + @Test + public void sign_withFlexibleChecksum_DelegatesToFlexibleChecksummer() { + SyncSignRequest request = generateBasicRequest( + AwsCredentialsIdentity.create("access", "secret"), + httpRequest -> { + }, + signRequest -> signRequest.putProperty(CHECKSUM_ALGORITHM, CRC32) + ); + + SyncSignedRequest signedRequest = signer.sign(request); + + assertThat(signedRequest.request().firstMatchingHeader("x-amz-checksum-crc32")).isPresent(); + assertThat(signedRequest.request().firstMatchingHeader("x-amz-content-sha256")).isPresent(); + } + + @Test + public void sign_withFlexibleChecksumAndPayloadSigningFalse_DelegatesToFlexibleChecksummer() { + SyncSignRequest request = generateBasicRequest( + AwsCredentialsIdentity.create("access", "secret"), + httpRequest -> { + }, + signRequest -> signRequest + .putProperty(CHECKSUM_ALGORITHM, CRC32) + .putProperty(PAYLOAD_SIGNING_ENABLED, false) + + ); + + SyncSignedRequest signedRequest = signer.sign(request); + + assertThat(signedRequest.request().firstMatchingHeader("x-amz-checksum-crc32")).isPresent(); + assertThat(signedRequest.request().firstMatchingHeader("x-amz-content-sha256")).hasValue("UNSIGNED-PAYLOAD"); + } } From 3124558336560ee4f3a18fbaf2b29181cc7f62a8 Mon Sep 17 00:00:00 2001 From: Hayden Baker Date: Thu, 31 Aug 2023 10:07:48 -0700 Subject: [PATCH 2/5] Add more tests, address comments --- core/http-auth-aws/pom.xml | 6 + .../internal/checksums/Crc32CChecksum.java | 15 +-- .../aws/internal/checksums/Crc32Checksum.java | 13 +- .../aws/internal/io/ChecksumInputStream.java | 3 + .../aws/internal/io/ChecksumSubscriber.java | 3 + .../signer/DefaultV4RequestSigner.java | 8 ++ .../internal/signer/FlexibleChecksummer.java | 4 +- .../signer/PrecomputedChecksummer.java | 4 +- .../auth/aws/internal/util/ChecksumUtil.java | 12 +- .../http/auth/aws/signer/Checksummer.java | 7 +- .../http/auth/aws/signer/V4Context.java | 8 +- .../signer/DefaultAwsV4HttpSignerTest.java | 5 +- .../signer/DefaultChecksummerTest.java | 64 ++++++++++ .../signer/DefaultRequestSignerTest.java | 69 ++++++++++ .../signer/FlexibleChecksummerTest.java | 120 ++++++++++++++++++ .../signer/PrecomputedChecksummerTest.java | 64 ++++++++++ .../aws/internal/util/ChecksumUtilTest.java | 66 ++++++++++ 17 files changed, 434 insertions(+), 37 deletions(-) create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultRequestSignerTest.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtilTest.java diff --git a/core/http-auth-aws/pom.xml b/core/http-auth-aws/pom.xml index e6f7a162aaa7..8a7c7a0a8d0f 100644 --- a/core/http-auth-aws/pom.xml +++ b/core/http-auth-aws/pom.xml @@ -69,6 +69,12 @@ checksums ${awsjavasdk.version} + + software.amazon.awssdk.crt + aws-crt + ${awscrt.version} + true + org.mockito diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java index 075f7dcb1044..7f625f2acbcb 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java @@ -17,10 +17,10 @@ import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.crt.checksums.CRC32C; import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32C; @@ -83,15 +83,10 @@ public void reset() { private Checksum cloneChecksum(Checksum checksum) { if (isCrtBasedChecksum) { - try { - Method method = checksum.getClass().getDeclaredMethod("clone"); - return (Checksum) method.invoke(checksum); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Could not clone checksum class " + checksum.getClass(), e); - } - } else { - return (Checksum) ((SdkCrc32C) checksum).clone(); - + return (Checksum) ((CRC32C) checksum).clone(); } + + return (Checksum) ((SdkCrc32C) checksum).clone(); + } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java index d5283d8ec359..dfc6575e3310 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java @@ -17,10 +17,10 @@ import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.crt.checksums.CRC32; import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32; @@ -81,14 +81,9 @@ public void reset() { private Checksum cloneChecksum(Checksum checksum) { if (isCrtBasedChecksum) { - try { - Method method = checksum.getClass().getDeclaredMethod("clone"); - return (Checksum) method.invoke(checksum); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Could not clone checksum class " + checksum.getClass(), e); - } - } else { - return (Checksum) ((SdkCrc32) checksum).clone(); + return (Checksum) ((CRC32) checksum).clone(); } + + return (Checksum) ((SdkCrc32) checksum).clone(); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java index 625af50d4b56..0ec6acbdd526 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStream.java @@ -23,6 +23,9 @@ import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; +/** + * An input-stream that takes a collection of checksums, and updates each checksum when it reads data. + */ @SdkInternalApi public class ChecksumInputStream extends FilterInputStream { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java index 072abe63a202..c7671f8c91e4 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriber.java @@ -25,6 +25,9 @@ import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; +/** + * A subscriber that takes a collection of checksums, and updates each checksum when it receives data. + */ @SdkInternalApi public final class ChecksumSubscriber implements Subscriber { private final CompletableFuture checksumming = new CompletableFuture<>(); diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java index d4763301855a..7e1a2850ccd4 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultV4RequestSigner.java @@ -32,6 +32,14 @@ import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.Logger; +/** + * The default implementation of a v4-request-signer. It performs each step of the SigV4 signing process, but does not add the + * signature or auth information to the request itself. + *

+ * All signing information, such as signature, signing key, canonical request, etc. is present in context object that is + * returned. This can be used by the caller to add the auth info to the request, such as adding the signature as a query + * parameter or building an authorization header using the signature and canonical request headers. + */ @SdkInternalApi public final class DefaultV4RequestSigner implements V4RequestSigner { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java index a7b2f56ea636..fac0b56f6d40 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java @@ -35,7 +35,9 @@ import software.amazon.awssdk.http.auth.aws.signer.Checksummer; /** - * A "flexible" implementation of a checksummer. + * A "flexible" implementation of a checksummer. It takes a map of checksums and their header names, computes them efficiently + * by updating each checksum while reading the payload (once), and adds the computed checksum strings to the request using the + * given header names in the map. This should be used in cases where a (flexible) checksum algorithm is present during signing. */ @SdkInternalApi public final class FlexibleChecksummer implements Checksummer { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java index e9adb3459f86..7a5fbb153ae8 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java @@ -28,8 +28,8 @@ /** * An implementation of a checksummer that simply passes along a computed value as a checksum. Specifically, this is used in - * the cases where the checksum is a pre-defined value that dictates specific behavior by the signer (such as aws-chunked - * payload signing, unsigned streaming with trailers, etc.). + * the cases where the checksum is a pre-defined value that dictates specific behavior by the signer, and flexible checksums is + * not enabled for the request (such as aws-chunked payload signing without trailers, unsigned streaming without trailers, etc.). */ @SdkInternalApi public final class PrecomputedChecksummer implements Checksummer { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java index c6e542b22f68..e96b6b69e18c 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java @@ -17,6 +17,7 @@ import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; @@ -27,6 +28,7 @@ import software.amazon.awssdk.http.auth.aws.internal.checksums.ConstantChecksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32CChecksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Md5Checksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha1Checksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; @@ -51,11 +53,7 @@ public static String checksumHeaderName(ChecksumAlgorithm checksumAlgorithm) { } /** - * Gets the SdkChecksum object based on the given ChecksumAlgorithm. Instances for CRC32C, CRC32 Algorithm will be added from - * CRT Java library once they are available in release. - * - * @param checksumAlgorithm Algorithm for calculating the checksum - * @return an SdkChecksum instance. + * Gets the SdkChecksum object based on the given ChecksumAlgorithm. */ public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { if (SHA256.algorithmId().equals(checksumAlgorithm.algorithmId())) { @@ -74,6 +72,10 @@ public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgori return new Crc32CChecksum(); } + if (MD5.algorithmId().equals(checksumAlgorithm.algorithmId())) { + return new Md5Checksum(); + } + if (CONSTANT_CHECKSUM.equals(checksumAlgorithm.algorithmId())) { return new ConstantChecksum(((ConstantChecksumAlgorithm) checksumAlgorithm).value); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java index c7baa06176bc..60531c93fecc 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java @@ -48,7 +48,7 @@ static Checksummer create() { } /** - * Get a flexible checksummer that uses the given checksum-algorithm and the default. + * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the default (sha256). */ static Checksummer create(ChecksumAlgorithm checksumAlgorithm) { if (checksumAlgorithm != null) { @@ -63,14 +63,15 @@ static Checksummer create(ChecksumAlgorithm checksumAlgorithm) { } /** - * Get a flexible checksummer that uses the given checksum-algorithm and the default. + * Get a precomputed checksummer that results the given checksum string. */ static Checksummer create(String checksum) { return new PrecomputedChecksummer(() -> checksum); } /** - * Get a flexible checksummer that uses the given checksum-algorithm and the default. + * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and a precomputed checksum from the + * given checksum string. */ static Checksummer create(String checksum, ChecksumAlgorithm checksumAlgorithm) { if (checksumAlgorithm != null) { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/V4Context.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/V4Context.java index f2f058c82715..6934c81e7c16 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/V4Context.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/V4Context.java @@ -25,7 +25,7 @@ @SdkProtectedApi @Immutable public final class V4Context { - private final String checksum; + private final String contentHash; private final byte[] signingKey; private final String signature; private final V4CanonicalRequest canonicalRequest; @@ -33,15 +33,15 @@ public final class V4Context { public V4Context(String contentHash, byte[] signingKey, String signature, V4CanonicalRequest canonicalRequest, SdkHttpRequest.Builder signedRequest) { - this.checksum = contentHash; + this.contentHash = contentHash; this.signingKey = signingKey.clone(); this.signature = signature; this.canonicalRequest = canonicalRequest; this.signedRequest = signedRequest; } - public String getContentChecksum() { - return checksum; + public String getContentHash() { + return contentHash; } public byte[] getSigningKey() { diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java index a0486c504cdd..446fd9a53e5a 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSignerTest.java @@ -235,7 +235,7 @@ public void sign_WithPayloadSigningFalseAndChunkEncodingTrueWithoutTrailer_Throw } @Test - public void sign_withFlexibleChecksum_DelegatesToFlexibleChecksummer() { + public void sign_withChecksumAlgorithm_DelegatesToChecksummerWithThatChecksumAlgorithm() { SyncSignRequest request = generateBasicRequest( AwsCredentialsIdentity.create("access", "secret"), httpRequest -> { @@ -250,7 +250,7 @@ public void sign_withFlexibleChecksum_DelegatesToFlexibleChecksummer() { } @Test - public void sign_withFlexibleChecksumAndPayloadSigningFalse_DelegatesToFlexibleChecksummer() { + public void sign_withChecksumAlgorithmAndPayloadSigningFalse_DelegatesToChecksummerWithThatChecksumAlgorithm() { SyncSignRequest request = generateBasicRequest( AwsCredentialsIdentity.create("access", "secret"), httpRequest -> { @@ -258,7 +258,6 @@ public void sign_withFlexibleChecksumAndPayloadSigningFalse_DelegatesToFlexibleC signRequest -> signRequest .putProperty(CHECKSUM_ALGORITHM, CRC32) .putProperty(PAYLOAD_SIGNING_ENABLED, false) - ); SyncSignedRequest signedRequest = signer.sign(request); diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java new file mode 100644 index 000000000000..f59f78295b32 --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.signer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.reactivex.Flowable; +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; + +public class DefaultChecksummerTest { + + ContentStreamProvider payload = () -> new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); + Publisher payloadAsync = Flowable.just(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8))); + + + SdkHttpRequest.Builder request = SdkHttpRequest.builder() + .uri(URI.create("https://localhost")) + .method(SdkHttpMethod.GET); + + @Test + public void checksummer_shouldAddSha256ChecksumToAmzContentSha256Header() { + DefaultChecksummer checksummer = new DefaultChecksummer(); + SdkHttpRequest expectedRequest = request + .putHeader("x-amz-content-sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .build(); + + checksummer.checksum(payload, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummerAsync_shouldAddSha256ChecksumToAmzContentSha256Header() { + DefaultChecksummer checksummer = new DefaultChecksummer(); + SdkHttpRequest expectedRequest = request + .putHeader("x-amz-content-sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .build(); + + checksummer.checksum(payloadAsync, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultRequestSignerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultRequestSignerTest.java new file mode 100644 index 000000000000..4cbe98387ca5 --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultRequestSignerTest.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.signer; + +import static java.time.ZoneOffset.UTC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static software.amazon.awssdk.utils.BinaryUtils.toHex; + +import java.net.URI; +import java.time.Clock; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.aws.signer.CredentialScope; +import software.amazon.awssdk.http.auth.aws.signer.V4Context; +import software.amazon.awssdk.http.auth.aws.signer.V4Properties; +import software.amazon.awssdk.http.auth.aws.signer.V4RequestSigner; +import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; + +public class DefaultRequestSignerTest { + + V4Properties v4Properties = V4Properties.builder() + .credentials(AwsCredentialsIdentity.create("foo", "bar")) + .credentialScope(new CredentialScope("baz", "qux", Instant.EPOCH)) + .signingClock(Clock.fixed(Instant.EPOCH, UTC)) + .doubleUrlEncode(true) + .normalizePath(true) + .build(); + + V4RequestSigner requestSigner = new DefaultV4RequestSigner(v4Properties); + + @Test + public void requestSigner_sign_shouldReturnSignedContext_butNotAddAnyAuthInfoToRequest() { + SdkHttpRequest.Builder request = SdkHttpRequest + .builder() + .uri(URI.create("https://localhost")) + .method(SdkHttpMethod.GET) + .putHeader("x-amz-content-sha256", "quux"); + String expectedContentHash = "quux"; + String expectedSigningKeyHex = "3d558b7a87b67996abc908071e0771a31b2a7977ab247144e60a6cba3356be1f"; + String expectedSignature = "7557839280ea0ef5c4acc66e5670d0857ac4f491884b1b8031d4dea2fc33483c"; + String expectedCanonicalRequestString = "GET\n/\n\n" + + "host:localhost\nx-amz-content-sha256:quux\n\n" + + "host;x-amz-content-sha256\nquux"; + String expectedHost = "localhost"; + + V4Context v4Context = requestSigner.sign(request); + + assertEquals(expectedContentHash, v4Context.getContentHash()); + assertEquals(expectedSigningKeyHex, toHex(v4Context.getSigningKey())); + assertEquals(expectedSignature, v4Context.getSignature()); + assertEquals(expectedCanonicalRequestString, v4Context.getCanonicalRequest().getCanonicalRequestString()); + assertEquals(expectedHost, v4Context.getSignedRequest().firstMatchingHeader("Host").orElse("")); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java new file mode 100644 index 000000000000..550fbc9ff003 --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java @@ -0,0 +1,120 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.signer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; + +import io.reactivex.Flowable; +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.utils.ImmutableMap; + +public class FlexibleChecksummerTest { + + ContentStreamProvider payload = () -> new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); + Publisher payloadAsync = Flowable.just(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8))); + + + SdkHttpRequest.Builder request = SdkHttpRequest.builder() + .uri(URI.create("https://localhost")) + .method(SdkHttpMethod.GET); + + @Test + public void checksummer_withNoChecksums_shouldNotAddAnyChecksum() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(Collections.emptyMap()); + SdkHttpRequest expectedRequest = request.build(); + + checksummer.checksum(payload, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummerAsync_withNoChecksums_shouldNotAddAnyChecksum() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(Collections.emptyMap()); + SdkHttpRequest expectedRequest = request.build(); + + checksummer.checksum(payloadAsync, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummer_withOneChecksum_shouldAddOneChecksum() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(ImmutableMap.of("sha256", SHA256)); + SdkHttpRequest expectedRequest = request + .putHeader("sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .build(); + + checksummer.checksum(payload, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummerAsync_withOneChecksum_shouldAddOneChecksum() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(Collections.emptyMap()); + SdkHttpRequest expectedRequest = request + .putHeader("sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .build(); + + checksummer.checksum(payloadAsync, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummer_withTwoChecksums_shouldAddTwoChecksums() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(ImmutableMap.of( + "sha256", SHA256, + "crc32", CRC32 + )); + SdkHttpRequest expectedRequest = request + .putHeader("sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .putHeader("crc32", "8c736521") + .build(); + + checksummer.checksum(payload, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummerAsync_withTwoChecksums_shouldAddTwoChecksums() { + FlexibleChecksummer checksummer = new FlexibleChecksummer(ImmutableMap.of( + "sha256", SHA256, + "crc32", CRC32 + )); + SdkHttpRequest expectedRequest = request + .putHeader("sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + .putHeader("crc32", "8c736521") + .build(); + + checksummer.checksum(payloadAsync, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java new file mode 100644 index 000000000000..ead8524019eb --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.signer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.reactivex.Flowable; +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; + +public class PrecomputedChecksummerTest { + + ContentStreamProvider payload = () -> new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); + Publisher payloadAsync = Flowable.just(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8))); + + + SdkHttpRequest.Builder request = SdkHttpRequest.builder() + .uri(URI.create("https://localhost")) + .method(SdkHttpMethod.GET); + + @Test + public void checksummer_shouldAddComputedValue() { + PrecomputedChecksummer checksummer = new PrecomputedChecksummer(() -> "something"); + SdkHttpRequest expectedRequest = request + .putHeader("x-amz-content-sha256", "something") + .build(); + + checksummer.checksum(payload, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } + + @Test + public void checksummerAsync_shouldAddComputedValue() { + PrecomputedChecksummer checksummer = new PrecomputedChecksummer(() -> "something"); + SdkHttpRequest expectedRequest = request + .putHeader("x-amz-content-sha256", "something") + .build(); + + checksummer.checksum(payloadAsync, request); + + assertEquals(expectedRequest.toString(), request.build().toString()); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtilTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtilTest.java new file mode 100644 index 000000000000..f0215c94b642 --- /dev/null +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtilTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.auth.aws.internal.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.checksumHeaderName; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.fromChecksumAlgorithm; +import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.readAll; + +import java.io.ByteArrayInputStream; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32CChecksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Crc32Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Md5Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha1Checksum; +import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; + +public class ChecksumUtilTest { + + @Test + public void checksumHeaderName_shouldFormatName() { + assertEquals("x-amz-checksum-sha256", checksumHeaderName(SHA256)); + assertEquals("x-amz-checksum-sha1", checksumHeaderName(SHA1)); + assertEquals("x-amz-checksum-crc32", checksumHeaderName(CRC32)); + assertEquals("x-amz-checksum-crc32c", checksumHeaderName(CRC32C)); + assertEquals("x-amz-checksum-md5", checksumHeaderName(MD5)); + } + + @Test + public void fromChecksumAlgorithm_mapsToCorrectSdkChecksum() { + assertEquals(Sha256Checksum.class, fromChecksumAlgorithm(SHA256).getClass()); + assertEquals(Sha1Checksum.class, fromChecksumAlgorithm(SHA1).getClass()); + assertEquals(Crc32Checksum.class, fromChecksumAlgorithm(CRC32).getClass()); + assertEquals(Crc32CChecksum.class, fromChecksumAlgorithm(CRC32C).getClass()); + assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); + } + + @Test + public void readAll_consumesEntireStream() { + int bytes = 4096 * 4 + 1; + byte[] buf = new byte[bytes]; + ByteArrayInputStream inputStream = new ByteArrayInputStream(buf); + + readAll(inputStream); + + assertEquals(0, inputStream.available()); + } +} From 60cda4e19bca08b8d2204e8cd5df4054ea560ffd Mon Sep 17 00:00:00 2001 From: Hayden Baker Date: Fri, 1 Sep 2023 10:28:09 -0700 Subject: [PATCH 3/5] Get rid of reflection, hasTrailer(), and address other comments --- .../internal/checksums/Crc32CChecksum.java | 40 +++++++--- .../aws/internal/checksums/Crc32Checksum.java | 41 +++++++--- .../SdkCrc32C.java => SdkCrc32CChecksum.java} | 14 ++-- .../SdkCrc32.java => SdkCrc32Checksum.java} | 14 ++-- .../factory/CrtBasedChecksumProvider.java | 78 ------------------- .../signer/DefaultAwsV4HttpSigner.java | 13 +--- .../internal/io/ChecksumInputStreamTest.java | 4 +- .../internal/io/ChecksumSubscriberTest.java | 9 +-- 8 files changed, 83 insertions(+), 130 deletions(-) rename core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/{factory/SdkCrc32C.java => SdkCrc32CChecksum.java} (98%) rename core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/{factory/SdkCrc32.java => SdkCrc32Checksum.java} (98%) delete mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java index 7f625f2acbcb..01a06f831d4f 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32CChecksum.java @@ -15,14 +15,12 @@ package software.amazon.awssdk.http.auth.aws.internal.checksums; -import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; - +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.crt.checksums.CRC32C; -import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; -import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32C; +import software.amazon.awssdk.utils.ClassLoaderHelper; /** * Implementation of {@link SdkChecksum} to calculate an CRC32C checksum. @@ -30,18 +28,19 @@ @SdkInternalApi public class Crc32CChecksum implements SdkChecksum { + private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C"; + private Checksum crc32c; private Checksum lastMarkedCrc32C; - private final boolean isCrtBasedChecksum; /** * Creates CRT Based Crc32C checksum if Crt classpath for Crc32c is loaded, else create Sdk Implemented Crc32c */ public Crc32CChecksum() { - crc32c = CrtBasedChecksumProvider.createCrc32C(); - isCrtBasedChecksum = crc32c != null; - if (!isCrtBasedChecksum) { - crc32c = SdkCrc32C.create(); + if (isCrtAvailable()) { + crc32c = new CRC32C(); + } else { + crc32c = SdkCrc32CChecksum.create(); } } @@ -82,11 +81,30 @@ public void reset() { private Checksum cloneChecksum(Checksum checksum) { - if (isCrtBasedChecksum) { + if (checksum instanceof CRC32C) { return (Checksum) ((CRC32C) checksum).clone(); } - return (Checksum) ((SdkCrc32C) checksum).clone(); + if (checksum instanceof SdkCrc32CChecksum) { + return (Checksum) ((SdkCrc32CChecksum) checksum).clone(); + } + + throw new IllegalStateException("Unsupported checksum"); + } + + public static boolean isCrtAvailable() { + try { + ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32C, false); + } catch (ClassNotFoundException e) { + return false; + } + + return true; + } + private static byte[] longToByte(Long input) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(input); + return buffer.array(); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java index dfc6575e3310..5faf3e9c660c 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/Crc32Checksum.java @@ -15,14 +15,12 @@ package software.amazon.awssdk.http.auth.aws.internal.checksums; -import static software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider.longToByte; - +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.crt.checksums.CRC32; -import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.CrtBasedChecksumProvider; -import software.amazon.awssdk.http.auth.aws.internal.checksums.factory.SdkCrc32; +import software.amazon.awssdk.utils.ClassLoaderHelper; /** * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum. @@ -30,18 +28,19 @@ @SdkInternalApi public class Crc32Checksum implements SdkChecksum { + private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32"; + private Checksum crc32; private Checksum lastMarkedCrc32; - private final boolean isCrtBasedChecksum; /** * Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32. */ public Crc32Checksum() { - crc32 = CrtBasedChecksumProvider.createCrc32(); - isCrtBasedChecksum = crc32 != null; - if (!isCrtBasedChecksum) { - crc32 = SdkCrc32.create(); + if (isCrtAvailable()) { + crc32 = new CRC32(); + } else { + crc32 = SdkCrc32Checksum.create(); } } @@ -80,10 +79,30 @@ public void reset() { } private Checksum cloneChecksum(Checksum checksum) { - if (isCrtBasedChecksum) { + if (checksum instanceof CRC32) { return (Checksum) ((CRC32) checksum).clone(); } - return (Checksum) ((SdkCrc32) checksum).clone(); + if (checksum instanceof SdkCrc32Checksum) { + return (Checksum) ((SdkCrc32Checksum) checksum).clone(); + } + + throw new IllegalStateException("Unsupported checksum"); + } + + public static boolean isCrtAvailable() { + try { + ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32, false); + } catch (ClassNotFoundException e) { + return false; + } + + return true; + } + + private static byte[] longToByte(Long input) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(input); + return buffer.array(); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32CChecksum.java similarity index 98% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java rename to core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32CChecksum.java index a408602ba1ae..88261fc48563 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32C.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32CChecksum.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; +package software.amazon.awssdk.http.auth.aws.internal.checksums; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -27,7 +27,7 @@ * The createCopy method is used to save current checksum state when the checksum is marked. */ @SdkInternalApi -public final class SdkCrc32C implements Checksum, Cloneable { +public final class SdkCrc32CChecksum implements Checksum, Cloneable { private static final int T8_0_START = 0 * 256; private static final int T8_1_START = 1 * 256; @@ -568,16 +568,16 @@ public final class SdkCrc32C implements Checksum, Cloneable { */ private int crc; - private SdkCrc32C() { + private SdkCrc32CChecksum() { reset(); } - private SdkCrc32C(int crc) { + private SdkCrc32CChecksum(int crc) { this.crc = crc; } - public static SdkCrc32C create() { - return new SdkCrc32C(); + public static SdkCrc32CChecksum create() { + return new SdkCrc32CChecksum(); } @Override @@ -633,6 +633,6 @@ public void update(int b) { @Override public Object clone() { - return new SdkCrc32C(crc); + return new SdkCrc32CChecksum(crc); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32Checksum.java similarity index 98% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java rename to core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32Checksum.java index db605a0373d4..8dc554b624ed 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/SdkCrc32.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/SdkCrc32Checksum.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; +package software.amazon.awssdk.http.auth.aws.internal.checksums; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -26,7 +26,7 @@ * The createCopy method is used to save current c checksum state when the checksum is marked. */ @SdkInternalApi -public final class SdkCrc32 implements Checksum, Cloneable { +public final class SdkCrc32Checksum implements Checksum, Cloneable { /* * CRC-32 lookup tables generated by the polynomial 0xEDB88320. @@ -559,18 +559,18 @@ public final class SdkCrc32 implements Checksum, Cloneable { */ private int crc; - private SdkCrc32() { + private SdkCrc32Checksum() { reset(); } - private SdkCrc32(int crc) { + private SdkCrc32Checksum(int crc) { this.crc = crc; } - public static SdkCrc32 create() { - return new SdkCrc32(); + public static SdkCrc32Checksum create() { + return new SdkCrc32Checksum(); } @Override @@ -617,6 +617,6 @@ public void update(int b) { @Override public Object clone() { - return new SdkCrc32(crc); + return new SdkCrc32Checksum(crc); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java deleted file mode 100644 index b184f48a7b27..000000000000 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/checksums/factory/CrtBasedChecksumProvider.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.checksums.factory; - -import java.nio.ByteBuffer; -import java.util.Optional; -import java.util.zip.Checksum; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.utils.ClassLoaderHelper; -import software.amazon.awssdk.utils.Lazy; -import software.amazon.awssdk.utils.Logger; - - -/** - * Class to load the Crt based checksum from aws-crt-java library if it is present in class path. - */ -@SdkInternalApi -public final class CrtBasedChecksumProvider { - - public static final Logger LOG = Logger.loggerFor(CrtBasedChecksumProvider.class); - private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C"; - private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32"; - private static final Lazy>> CRT_CRC32_CLASS_LOADER = - new Lazy<>(() -> initializeCrtChecksumClass(CRT_CLASSPATH_FOR_CRC32)); - private static final Lazy>> CRT_CRC32_C_CLASS_LOADER = - new Lazy<>(() -> initializeCrtChecksumClass(CRT_CLASSPATH_FOR_CRC32C)); - - private CrtBasedChecksumProvider() { - } - - public static Checksum createCrc32() { - return createCrtBasedChecksum(CRT_CRC32_CLASS_LOADER); - } - - public static Checksum createCrc32C() { - return createCrtBasedChecksum(CRT_CRC32_C_CLASS_LOADER); - } - - private static Checksum createCrtBasedChecksum(Lazy>> lazyClassLoader) { - return lazyClassLoader.getValue().map( - checksumClass -> { - try { - return (Checksum) checksumClass.getDeclaredConstructor().newInstance(); - } catch (ReflectiveOperationException e) { - return null; - } - }).orElse(null); - } - - private static Optional> initializeCrtChecksumClass(String classPath) { - try { - return Optional.of(ClassLoaderHelper.loadClass(classPath, false)); - } catch (ClassNotFoundException e) { - LOG.debug(() -> "Cannot find the " + classPath + " class." - + " To invoke a request that requires a CRT based checksums.", e); - return Optional.empty(); - } - } - - public static byte[] longToByte(Long input) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(input); - return buffer.array(); - } -} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index 6b155b52e7ff..957e1092c02f 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -111,7 +111,8 @@ private static Checksummer checksummer(SignRequest STREAMING_EVENTS_PAYLOAD); @@ -119,7 +120,7 @@ private static Checksummer checksummer(SignRequest STREAMING_SIGNED_PAYLOAD_TRAILER); } return new PrecomputedChecksummer(() -> STREAMING_SIGNED_PAYLOAD); @@ -128,7 +129,7 @@ private static Checksummer checksummer(SignRequest STREAMING_UNSIGNED_PAYLOAD_TRAILER); } throw new UnsupportedOperationException("Chunk-Encoding without Payload-Signing must have a trailer!"); @@ -230,12 +231,6 @@ private static boolean isEventStreaming(SdkHttpRequest request) { return "application/vnd.amazon.eventstream".equals(request.firstMatchingHeader(Header.CONTENT_TYPE).orElse("")); } - private static boolean hasTrailer(SignRequest request) { - // Flexible checksums dictates adding a trailer when signing, but there may be existing trailers on - // the request (in the future) - return request.hasProperty(CHECKSUM_ALGORITHM) || request.request().firstMatchingHeader(X_AMZ_TRAILER).isPresent(); - } - /** * A class-loader for the event-stream signer, which throws exceptions if it can't load the class (it's likely not on the * classpath, so it should be added), or if it can't instantiate the signer. diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java index a848bf6fa944..bd9ce32c8193 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumInputStreamTest.java @@ -31,7 +31,7 @@ public class ChecksumInputStreamTest { @Test - public void test_computesCorrectSha256() { + public void read_computesCorrectSha256() { String testString = "AWS SDK for Java"; String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; @@ -46,7 +46,7 @@ public void test_computesCorrectSha256() { } @Test - public void test_withMultipleChecksums_shouldComputeCorrectChecksums() { + public void read_withMultipleChecksums_shouldComputeCorrectChecksums() { String testString = "AWS SDK for Java"; String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; String expectedCrc32Digest = "4ac37ece"; diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java index 4be08fadbb4d..202c31d05a61 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/io/ChecksumSubscriberTest.java @@ -24,7 +24,6 @@ import static software.amazon.awssdk.utils.CompletableFutureUtils.joinLikeSync; import io.reactivex.Flowable; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -39,7 +38,7 @@ public class ChecksumSubscriberTest { @Test - public void test_computesCorrectSha256() { + public void checksum_computesCorrectSha256() { String testString = "AWS SDK for Java"; String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; @@ -55,7 +54,7 @@ public void test_computesCorrectSha256() { } @Test - public void test_withMultipleChecksums_shouldComputeCorrectChecksums() { + public void checksum_withMultipleChecksums_shouldComputeCorrectChecksums() { String testString = "AWS SDK for Java"; String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; String expectedCrc32Digest = "4ac37ece"; @@ -75,7 +74,7 @@ public void test_withMultipleChecksums_shouldComputeCorrectChecksums() { } @Test - public void test_futureCancelledBeforeSubscribe_cancelsSubscription() { + public void checksum_futureCancelledBeforeSubscribe_cancelsSubscription() { Subscription mockSubscription = mock(Subscription.class); ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.emptyList()); @@ -89,7 +88,7 @@ public void test_futureCancelledBeforeSubscribe_cancelsSubscription() { } @Test - public void test_publisherCallsOnError_errorPropagatedToFuture() { + public void checksum_publisherCallsOnError_errorPropagatedToFuture() { Subscription mockSubscription = mock(Subscription.class); ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.emptyList()); From dd67e7c725ebd2f79c49d6aa93a7bbbdf4723969 Mon Sep 17 00:00:00 2001 From: Hayden Baker Date: Tue, 5 Sep 2023 15:24:49 -0700 Subject: [PATCH 4/5] Remove DefaultChecksummer, update namings --- .../signer/DefaultAwsV4HttpSigner.java | 18 +++-- .../internal/signer/DefaultChecksummer.java | 73 ------------------- .../internal/signer/FlexibleChecksummer.java | 14 ++-- ...java => PrecomputedSha256Checksummer.java} | 4 +- .../auth/aws/internal/util/ChecksumUtil.java | 31 ++++---- .../http/auth/aws/signer/Checksummer.java | 32 ++++---- .../signer/DefaultChecksummerTest.java | 64 ---------------- ... => PrecomputedSha256ChecksummerTest.java} | 6 +- 8 files changed, 54 insertions(+), 188 deletions(-) delete mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java rename core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/{PrecomputedChecksummer.java => PrecomputedSha256Checksummer.java} (94%) delete mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java rename core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/{PrecomputedChecksummerTest.java => PrecomputedSha256ChecksummerTest.java} (89%) diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index 957e1092c02f..c8b8caea20b3 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -61,7 +61,7 @@ public final class DefaultAwsV4HttpSigner implements AwsV4HttpSigner { private static final int DEFAULT_CHUNK_SIZE_IN_BYTES = 128 * 1024; private static V4Properties v4Properties(SignRequest request) { - Clock signingClock = request.requireProperty(AwsV4HttpSigner.SIGNING_CLOCK, Clock.systemUTC()); + Clock signingClock = request.requireProperty(SIGNING_CLOCK, Clock.systemUTC()); Instant signingInstant = signingClock.instant(); AwsCredentialsIdentity credentials = sanitizeCredentials(request.identity()); String regionName = request.requireProperty(AwsV4HttpSigner.REGION_NAME); @@ -115,27 +115,31 @@ private static Checksummer checksummer(SignRequest STREAMING_EVENTS_PAYLOAD); + return Checksummer.forPrecomputed256Checksum(STREAMING_EVENTS_PAYLOAD); } if (isPayloadSigning) { if (isChunkEncoding) { if (isFlexible || isTrailing) { - return new PrecomputedChecksummer(() -> STREAMING_SIGNED_PAYLOAD_TRAILER); + return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD_TRAILER); } - return new PrecomputedChecksummer(() -> STREAMING_SIGNED_PAYLOAD); + return Checksummer.forPrecomputed256Checksum(STREAMING_SIGNED_PAYLOAD); } - return Checksummer.create(request.property(CHECKSUM_ALGORITHM)); + + if (request.hasProperty(CHECKSUM_ALGORITHM)) { + return Checksummer.forFlexibleChecksum(request.property(CHECKSUM_ALGORITHM)); + } + return Checksummer.create(); } if (isChunkEncoding) { if (isFlexible || isTrailing) { - return new PrecomputedChecksummer(() -> STREAMING_UNSIGNED_PAYLOAD_TRAILER); + return Checksummer.forPrecomputed256Checksum(STREAMING_UNSIGNED_PAYLOAD_TRAILER); } throw new UnsupportedOperationException("Chunk-Encoding without Payload-Signing must have a trailer!"); } - return Checksummer.create(UNSIGNED_PAYLOAD, request.property(CHECKSUM_ALGORITHM)); + return Checksummer.forFlexibleChecksum(UNSIGNED_PAYLOAD, request.property(CHECKSUM_ALGORITHM)); } private static V4PayloadSigner v4PayloadSigner( diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java deleted file mode 100644 index 1760024ff066..000000000000 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummer.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.signer; - -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; -import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.fromChecksumAlgorithm; -import static software.amazon.awssdk.http.auth.aws.internal.util.ChecksumUtil.readAll; -import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; -import static software.amazon.awssdk.http.auth.aws.util.SignerUtils.getBinaryRequestPayloadStream; -import static software.amazon.awssdk.utils.BinaryUtils.toHex; - -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.concurrent.CompletableFuture; -import org.reactivestreams.Publisher; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.ContentStreamProvider; -import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; -import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumInputStream; -import software.amazon.awssdk.http.auth.aws.internal.io.ChecksumSubscriber; -import software.amazon.awssdk.http.auth.aws.signer.Checksummer; - -/** - * The default implementation of a checksummer. By default, this will calculate the SHA256 checksum of a payload and add it as the - * value for the 'x-amz-content-sha256' header on the request. - */ -@SdkInternalApi -public final class DefaultChecksummer implements Checksummer { - - @Override - public void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder request) { - SdkChecksum sdkChecksum = fromChecksumAlgorithm(SHA256); - InputStream payloadStream = new ChecksumInputStream( - getBinaryRequestPayloadStream(payload), - Collections.singletonList(sdkChecksum) - ); - - readAll(payloadStream); - - request.putHeader(X_AMZ_CONTENT_SHA256, toHex(sdkChecksum.getChecksumBytes())); - } - - @Override - public CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request) { - SdkChecksum sdkChecksum = fromChecksumAlgorithm(SHA256); - - ChecksumSubscriber checksumSubscriber = new ChecksumSubscriber(Collections.singletonList(sdkChecksum)); - - if (payload != null) { - payload.subscribe(checksumSubscriber); - } - - return checksumSubscriber.checksum().thenRun(() -> { - String checksum = toHex(sdkChecksum.getChecksumBytes()); - request.putHeader(X_AMZ_CONTENT_SHA256, checksum); - }); - } -} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java index fac0b56f6d40..b0c5cdfa002c 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java @@ -41,10 +41,10 @@ */ @SdkInternalApi public final class FlexibleChecksummer implements Checksummer { - private final Map checksumMap; + private final Map headerToChecksum; - public FlexibleChecksummer(Map algorithmMap) { - this.checksumMap = algorithmMap + public FlexibleChecksummer(Map headerToChecksumAlgorithm) { + this.headerToChecksum = headerToChecksumAlgorithm .entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, v -> fromChecksumAlgorithm(v.getValue()))); @@ -56,24 +56,24 @@ public void checksum(ContentStreamProvider payload, SdkHttpRequest.Builder reque ChecksumInputStream computingStream = new ChecksumInputStream( payloadStream, - checksumMap.values() + headerToChecksum.values() ); readAll(computingStream); - checksumMap.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())); + headerToChecksum.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())); } @Override public CompletableFuture checksum(Publisher payload, SdkHttpRequest.Builder request) { - ChecksumSubscriber checksumSubscriber = new ChecksumSubscriber(checksumMap.values()); + ChecksumSubscriber checksumSubscriber = new ChecksumSubscriber(headerToChecksum.values()); if (payload != null) { payload.subscribe(checksumSubscriber); } return checksumSubscriber.checksum().thenRun( - () -> checksumMap.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())) + () -> headerToChecksum.forEach((header, checksum) -> request.putHeader(header, checksum.getChecksum())) ); } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256Checksummer.java similarity index 94% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java rename to core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256Checksummer.java index 7a5fbb153ae8..351b757275b5 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256Checksummer.java @@ -32,11 +32,11 @@ * not enabled for the request (such as aws-chunked payload signing without trailers, unsigned streaming without trailers, etc.). */ @SdkInternalApi -public final class PrecomputedChecksummer implements Checksummer { +public final class PrecomputedSha256Checksummer implements Checksummer { private final Callable computation; - public PrecomputedChecksummer(Callable computation) { + public PrecomputedSha256Checksummer(Callable computation) { this.computation = computation; } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java index e96b6b69e18c..fb88ee14310e 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/util/ChecksumUtil.java @@ -23,6 +23,8 @@ import java.io.InputStream; import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.auth.aws.internal.checksums.ConstantChecksum; @@ -32,12 +34,21 @@ import software.amazon.awssdk.http.auth.aws.internal.checksums.SdkChecksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha1Checksum; import software.amazon.awssdk.http.auth.aws.internal.checksums.Sha256Checksum; +import software.amazon.awssdk.utils.ImmutableMap; @SdkInternalApi public final class ChecksumUtil { private static final String CONSTANT_CHECKSUM = "CONSTANT"; + private static final Map> checksumMap = ImmutableMap.of( + SHA256.algorithmId(), Sha256Checksum::new, + SHA1.algorithmId(), Sha1Checksum::new, + CRC32.algorithmId(), Crc32Checksum::new, + CRC32C.algorithmId(), Crc32CChecksum::new, + MD5.algorithmId(), Md5Checksum::new + ); + private ChecksumUtil() { } @@ -56,24 +67,8 @@ public static String checksumHeaderName(ChecksumAlgorithm checksumAlgorithm) { * Gets the SdkChecksum object based on the given ChecksumAlgorithm. */ public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { - if (SHA256.algorithmId().equals(checksumAlgorithm.algorithmId())) { - return new Sha256Checksum(); - } - - if (SHA1.algorithmId().equals(checksumAlgorithm.algorithmId())) { - return new Sha1Checksum(); - } - - if (CRC32.algorithmId().equals(checksumAlgorithm.algorithmId())) { - return new Crc32Checksum(); - } - - if (CRC32C.algorithmId().equals(checksumAlgorithm.algorithmId())) { - return new Crc32CChecksum(); - } - - if (MD5.algorithmId().equals(checksumAlgorithm.algorithmId())) { - return new Md5Checksum(); + if (checksumMap.containsKey(checksumAlgorithm.algorithmId())) { + return checksumMap.get(checksumAlgorithm.algorithmId()).get(); } if (CONSTANT_CHECKSUM.equals(checksumAlgorithm.algorithmId())) { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java index 60531c93fecc..d4f3f05c937d 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java @@ -21,6 +21,7 @@ import static software.amazon.awssdk.http.auth.aws.util.SignerConstant.X_AMZ_CONTENT_SHA256; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import org.reactivestreams.Publisher; @@ -28,9 +29,8 @@ import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.http.auth.aws.internal.signer.DefaultChecksummer; import software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer; -import software.amazon.awssdk.http.auth.aws.internal.signer.PrecomputedChecksummer; +import software.amazon.awssdk.http.auth.aws.internal.signer.PrecomputedSha256Checksummer; import software.amazon.awssdk.utils.ImmutableMap; /** @@ -41,16 +41,18 @@ @SdkProtectedApi public interface Checksummer { /** - * Get a default implementation of a checksummer. + * Get a default implementation of a checksummer, which calculates the SHA-256 checksum and places it in the + * x-amz-content-sha256 header. */ static Checksummer create() { - return new DefaultChecksummer(); + return new FlexibleChecksummer(Collections.singletonMap(X_AMZ_CONTENT_SHA256, SHA256)); } /** - * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the default (sha256). + * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the SHA-256 checksum. It places + * the SHA-256 checksum in x-amz-content-sha256 header, and the given checksum-algorithm in the x-amz-checksum-[name] header. */ - static Checksummer create(ChecksumAlgorithm checksumAlgorithm) { + static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm) { if (checksumAlgorithm != null) { Map checksums = ImmutableMap.of( X_AMZ_CONTENT_SHA256, SHA256, @@ -59,30 +61,32 @@ static Checksummer create(ChecksumAlgorithm checksumAlgorithm) { return new FlexibleChecksummer(checksums); } - return create(); + + throw new IllegalArgumentException("Checksum Algorithm cannot be null!"); } /** - * Get a precomputed checksummer that results the given checksum string. + * Get a precomputed checksummer which places the precomputed checksum to the x-amz-content-sha256 header. */ - static Checksummer create(String checksum) { - return new PrecomputedChecksummer(() -> checksum); + static Checksummer forPrecomputed256Checksum(String precomputedSha256) { + return new PrecomputedSha256Checksummer(() -> precomputedSha256); } /** * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and a precomputed checksum from the - * given checksum string. + * given checksum string. It places the precomputed checksum in x-amz-content-sha256 header, and the given checksum-algorithm + * in the x-amz-checksum-[name] header. */ - static Checksummer create(String checksum, ChecksumAlgorithm checksumAlgorithm) { + static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorithm checksumAlgorithm) { if (checksumAlgorithm != null) { Map checksums = ImmutableMap.of( - X_AMZ_CONTENT_SHA256, new ConstantChecksumAlgorithm(checksum), + X_AMZ_CONTENT_SHA256, new ConstantChecksumAlgorithm(precomputedSha256), checksumHeaderName(checksumAlgorithm), checksumAlgorithm ); return new FlexibleChecksummer(checksums); } - return create(checksum); + return forPrecomputed256Checksum(precomputedSha256); } /** diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java deleted file mode 100644 index f59f78295b32..000000000000 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultChecksummerTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.signer; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.reactivex.Flowable; -import java.io.ByteArrayInputStream; -import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import org.junit.jupiter.api.Test; -import org.reactivestreams.Publisher; -import software.amazon.awssdk.http.ContentStreamProvider; -import software.amazon.awssdk.http.SdkHttpMethod; -import software.amazon.awssdk.http.SdkHttpRequest; - -public class DefaultChecksummerTest { - - ContentStreamProvider payload = () -> new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); - Publisher payloadAsync = Flowable.just(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8))); - - - SdkHttpRequest.Builder request = SdkHttpRequest.builder() - .uri(URI.create("https://localhost")) - .method(SdkHttpMethod.GET); - - @Test - public void checksummer_shouldAddSha256ChecksumToAmzContentSha256Header() { - DefaultChecksummer checksummer = new DefaultChecksummer(); - SdkHttpRequest expectedRequest = request - .putHeader("x-amz-content-sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") - .build(); - - checksummer.checksum(payload, request); - - assertEquals(expectedRequest.toString(), request.build().toString()); - } - - @Test - public void checksummerAsync_shouldAddSha256ChecksumToAmzContentSha256Header() { - DefaultChecksummer checksummer = new DefaultChecksummer(); - SdkHttpRequest expectedRequest = request - .putHeader("x-amz-content-sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") - .build(); - - checksummer.checksum(payloadAsync, request); - - assertEquals(expectedRequest.toString(), request.build().toString()); - } -} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256ChecksummerTest.java similarity index 89% rename from core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java rename to core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256ChecksummerTest.java index ead8524019eb..a25d95b5e282 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedChecksummerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/PrecomputedSha256ChecksummerTest.java @@ -28,7 +28,7 @@ import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; -public class PrecomputedChecksummerTest { +public class PrecomputedSha256ChecksummerTest { ContentStreamProvider payload = () -> new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8)); Publisher payloadAsync = Flowable.just(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8))); @@ -40,7 +40,7 @@ public class PrecomputedChecksummerTest { @Test public void checksummer_shouldAddComputedValue() { - PrecomputedChecksummer checksummer = new PrecomputedChecksummer(() -> "something"); + PrecomputedSha256Checksummer checksummer = new PrecomputedSha256Checksummer(() -> "something"); SdkHttpRequest expectedRequest = request .putHeader("x-amz-content-sha256", "something") .build(); @@ -52,7 +52,7 @@ public void checksummer_shouldAddComputedValue() { @Test public void checksummerAsync_shouldAddComputedValue() { - PrecomputedChecksummer checksummer = new PrecomputedChecksummer(() -> "something"); + PrecomputedSha256Checksummer checksummer = new PrecomputedSha256Checksummer(() -> "something"); SdkHttpRequest expectedRequest = request .putHeader("x-amz-content-sha256", "something") .build(); From a3437f7e25ca338cba915d4bde0fb99ed212f6a3 Mon Sep 17 00:00:00 2001 From: Hayden Baker Date: Wed, 6 Sep 2023 08:33:10 -0700 Subject: [PATCH 5/5] Address last comments --- .../auth/aws/internal/signer/DefaultAwsV4HttpSigner.java | 6 +++++- .../awssdk/http/auth/aws/internal/util/ChecksumUtil.java | 6 +++--- .../amazon/awssdk/http/auth/aws/signer/Checksummer.java | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index c8b8caea20b3..7b75e6c36447 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -139,7 +139,11 @@ private static Checksummer checksummer(SignRequest> checksumMap = ImmutableMap.of( + private static final Map> CHECKSUM_MAP = ImmutableMap.of( SHA256.algorithmId(), Sha256Checksum::new, SHA1.algorithmId(), Sha1Checksum::new, CRC32.algorithmId(), Crc32Checksum::new, @@ -67,8 +67,8 @@ public static String checksumHeaderName(ChecksumAlgorithm checksumAlgorithm) { * Gets the SdkChecksum object based on the given ChecksumAlgorithm. */ public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { - if (checksumMap.containsKey(checksumAlgorithm.algorithmId())) { - return checksumMap.get(checksumAlgorithm.algorithmId()).get(); + if (CHECKSUM_MAP.containsKey(checksumAlgorithm.algorithmId())) { + return CHECKSUM_MAP.get(checksumAlgorithm.algorithmId()).get(); } if (CONSTANT_CHECKSUM.equals(checksumAlgorithm.algorithmId())) { diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java index d4f3f05c937d..ce08e89613f2 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/signer/Checksummer.java @@ -86,7 +86,8 @@ X_AMZ_CONTENT_SHA256, new ConstantChecksumAlgorithm(precomputedSha256), return new FlexibleChecksummer(checksums); } - return forPrecomputed256Checksum(precomputedSha256); + + throw new IllegalArgumentException("Checksum Algorithm cannot be null!"); } /**