diff --git a/.changes/next-release/deprecation-AWSSDKforJavav2-eb641b5.json b/.changes/next-release/deprecation-AWSSDKforJavav2-eb641b5.json new file mode 100644 index 000000000000..7ba25e79d5c2 --- /dev/null +++ b/.changes/next-release/deprecation-AWSSDKforJavav2-eb641b5.json @@ -0,0 +1,6 @@ +{ + "type": "deprecation", + "category": "AWS SDK for Java v2", + "contributor": "", + "description": "Deprecate internal checksum algorithm classes." +} diff --git a/.changes/next-release/feature-AWSSDKforJavav2-3676574.json b/.changes/next-release/feature-AWSSDKforJavav2-3676574.json new file mode 100644 index 000000000000..ac072611aa20 --- /dev/null +++ b/.changes/next-release/feature-AWSSDKforJavav2-3676574.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS SDK for Java v2", + "contributor": "", + "description": "The SDK now defaults to Java built-in CRC32 and CRC32C(if it's Java 9+) implementations, resulting in improved performance." +} diff --git a/build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml b/build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml index b1f14527e131..e123f641d076 100644 --- a/build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml +++ b/build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml @@ -303,8 +303,8 @@ - - + + diff --git a/core/checksums/pom.xml b/core/checksums/pom.xml index 4f96a2aeb58e..19f14ac7377b 100644 --- a/core/checksums/pom.xml +++ b/core/checksums/pom.xml @@ -45,6 +45,19 @@ ${awsjavasdk.version} + + software.amazon.awssdk + utils + ${awsjavasdk.version} + + + + software.amazon.awssdk.crt + aws-crt + ${awscrt.version} + true + + org.junit.jupiter junit-jupiter diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java index b83e049bac80..10a50c2e704f 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java @@ -29,6 +29,7 @@ public final class DefaultChecksumAlgorithm { public static final ChecksumAlgorithm MD5 = of("MD5"); public static final ChecksumAlgorithm SHA256 = of("SHA256"); public static final ChecksumAlgorithm SHA1 = of("SHA1"); + public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME"); private DefaultChecksumAlgorithm() { } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java similarity index 71% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkChecksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java index ab7b5a388e4d..12d5025f82ea 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java @@ -13,18 +13,48 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums; import java.nio.ByteBuffer; import java.util.zip.Checksum; -import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.checksums.internal.Crc32Checksum; +import software.amazon.awssdk.checksums.internal.Crc32cProvider; +import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; +import software.amazon.awssdk.checksums.internal.Md5Checksum; +import software.amazon.awssdk.checksums.internal.Sha1Checksum; +import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; /** * Extension of {@link Checksum} to support checksums and checksum validations used by the SDK that are not provided by the JDK. */ -@SdkInternalApi +@SdkProtectedApi public interface SdkChecksum extends Checksum { + /** + * Returns an {@link SdkChecksum} based on the {@link ChecksumAlgorithm} provided. + * UnsupportedOperationException will be thrown for unsupported algorithm. + */ + static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) { + switch (algorithm.algorithmId()) { + case "CRC32C": + return Crc32cProvider.create(); + case "CRC32": + return new Crc32Checksum(); + case "SHA1": + return new Sha1Checksum(); + case "SHA256": + return new Sha256Checksum(); + case "MD5": + return new Md5Checksum(); + case "CRC64NVME": + return new Crc64NvmeChecksum(); + default: + throw new UnsupportedOperationException("Unsupported checksum algorithm: " + algorithm); + } + } + /** * Returns the computed checksum in a byte array rather than the long provided by {@link #getValue()}. * @@ -49,7 +79,6 @@ default void update(byte[] b) { update(b, 0, b.length); } - /** * Updates the current checksum with the bytes from the specified buffer. *

diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/BaseCrcChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/BaseCrcChecksum.java new file mode 100644 index 000000000000..174facf8448c --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/BaseCrcChecksum.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; + +/** + * Base class for CRC related checksums + */ +@SdkInternalApi +public abstract class BaseCrcChecksum implements SdkChecksum { + + private Checksum checksum; + private Checksum lastMarkedChecksum; + + public BaseCrcChecksum(Checksum checksum) { + this.checksum = checksum; + } + + public Checksum getChecksum() { + return checksum; + } + + @Override + public void mark(int readLimit) { + this.lastMarkedChecksum = cloneChecksum(checksum); + } + + @Override + public void update(int b) { + checksum.update(b); + } + + @Override + public void update(byte[] b, int off, int len) { + checksum.update(b, off, len); + } + + @Override + public long getValue() { + return checksum.getValue(); + } + + @Override + public void reset() { + if (lastMarkedChecksum == null) { + checksum.reset(); + } else { + checksum = cloneChecksum(lastMarkedChecksum); + } + } + + abstract Checksum cloneChecksum(Checksum checksum); +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java new file mode 100644 index 000000000000..a710ed0a0ef4 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java @@ -0,0 +1,83 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.lang.ref.WeakReference; +import java.lang.reflect.Constructor; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.ClassLoaderHelper; +import software.amazon.awssdk.utils.Logger; + +/** + * A cache that stores classes and their constructors by class name and class loader. + *

+ * This cache uses weak references to both class loaders and classes, allowing them to be garbage collected + * when no longer needed. It provides methods to retrieve the zero-argument constructor for a class, + * based on the current thread's context class loader or the system class loader. + *

+ * If a class or its zero-argument constructor cannot be found, an empty result is returned. + */ +@SdkInternalApi +public final class ConstructorCache { + private static final Logger log = Logger.loggerFor(ConstructorCache.class); + + /** + * Cache storing classes by class name and class loader. + * Uses weak references to allow garbage collection when not needed. + */ + private final Map>>>> classesByClassName = + new ConcurrentHashMap<>(); + + /** + * Retrieve the class for the given class name from the context or system class loader. + * Returns an empty result if the class is not found. + */ + private Optional> getClass(String className) { + Map>>> classesByClassLoader = + classesByClassName.computeIfAbsent(className, k -> Collections.synchronizedMap(new WeakHashMap<>())); + + ClassLoader classLoader = ClassLoaderHelper.contextClassLoader(); + Optional>> classRef = classesByClassLoader.computeIfAbsent(classLoader, k -> { + try { + Class clazz = classLoader.loadClass(className); + return Optional.of(new WeakReference<>(clazz)); + } catch (ClassNotFoundException e) { + return Optional.empty(); + } + }); + return classRef.map(WeakReference::get); + } + + /** + * Retrieve the zero-argument constructor for the given class name. + * Returns an empty result if no such constructor is found. + */ + public Optional> getConstructor(String className) { + return getClass(className).flatMap(clazz -> { + try { + return Optional.of(clazz.getConstructor()); + } catch (NoSuchMethodException e) { + log.debug(() -> "Classloader contains " + className + ", but without a zero-arg constructor.", e); + return Optional.empty(); + } + }); + } +} \ No newline at end of file diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java new file mode 100644 index 000000000000..ac93a3e7f4e1 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + + +import java.util.zip.CRC32; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; + +@SdkInternalApi +public final class Crc32Checksum implements SdkChecksum { + private final CrcCombineOnMarkChecksum crc32; + + public Crc32Checksum() { + // Delegates to CrcCombineOnMarkChecksum with CRC32 + this.crc32 = new CrcCombineOnMarkChecksum( + new CRC32(), + SdkCrc32Checksum::combine + ); + } + + @Override + public byte[] getChecksumBytes() { + return crc32.getChecksumBytes(); + } + + @Override + public void mark(int readLimit) { + crc32.mark(readLimit); + } + + @Override + public void update(int b) { + crc32.update(b); + } + + @Override + public void update(byte[] b, int off, int len) { + crc32.update(b, off, len); + } + + @Override + public long getValue() { + return crc32.getValue(); + } + + @Override + public void reset() { + crc32.reset(); + } +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java new file mode 100644 index 000000000000..9b35f043a0f3 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java @@ -0,0 +1,86 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.crt.checksums.CRC32C; + +/** + * Utility class to provide different implementations of CRC32C checksum. This class supports the use of: 1. Java-based CRC32C + * (Java 9+ when available) 2. CRT-based CRC32C (when available) 3. SDK-based CRC32C (as fallback) + */ +@SdkInternalApi +public final class Crc32cProvider { + + + // Class paths for different CRC32C implementations + private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C"; + private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C"; + private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache(); + + // Private constructor to prevent instantiation + private Crc32cProvider() { + } + + /** + * Creates an instance of the SDK-based CRC32C checksum as a fallback. + * + * @return An SdkChecksum instance. + */ + static SdkChecksum createSdkBasedCrc32C() { + SdkCrc32CChecksum sdkChecksum = SdkCrc32CChecksum.create(); + return new CrcCloneOnMarkChecksum(sdkChecksum, checksumToClone -> ((SdkCrc32CChecksum) checksumToClone).clone()); + } + + /** + * Tries to create a Java 9-based CRC32C checksum. + * If it's not available, it tries to create a CRT-based checksum. + * If both are not available, it falls back to an SDK-based CRC32C checksum. + * + * @return An instance of {@link SdkChecksum}, based on the first available option. + */ + public static SdkChecksum create() { + SdkChecksum checksum = createJavaCrc32C(); + if (checksum == null) { + checksum = createCrtCrc32C(); + } + return checksum != null ? checksum : createSdkBasedCrc32C(); + } + + static SdkChecksum createCrtCrc32C() { + return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> { + try { + return new CrcCloneOnMarkChecksum((Checksum) constructor.newInstance(), checksumToClone -> + (Checksum) ((CRC32C) checksumToClone).clone()); + } catch (ClassCastException | ReflectiveOperationException e) { + throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e); + } + }).orElse(null); + } + + static SdkChecksum createJavaCrc32C() { + return CONSTRUCTOR_CACHE.getConstructor(JAVA_CRC32C_CLASS_PATH).map(constructor -> { + try { + return new CrcCombineOnMarkChecksum((Checksum) constructor.newInstance(), SdkCrc32CChecksum::combine); + } catch (ClassCastException | ReflectiveOperationException e) { + throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e); + } + }).orElse(null); + } + +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java new file mode 100644 index 000000000000..56dea2775793 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static software.amazon.awssdk.utils.DependencyValidate.requireClass; +import static software.amazon.awssdk.utils.NumericUtils.longToByte; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.crt.checksums.CRC64NVME; + +/** + * Implementation of {@link SdkChecksum} to calculate an CRC64NVME checksum. + */ +@SdkInternalApi +public final class Crc64NvmeChecksum extends BaseCrcChecksum { + private static final String CRT_CRC64NVME_PATH = "software.amazon.awssdk.crt.checksums.CRC64NVME"; + private static final String CRT_MODULE = "software.amazon.awssdk.crt:aws-crt"; + + public Crc64NvmeChecksum() { + super(getCrc64Nvme()); + } + + private static CRC64NVME getCrc64Nvme() { + requireClass(CRT_CRC64NVME_PATH, CRT_MODULE, "CRC64NVME"); + return new CRC64NVME(); + } + + @Override + public Checksum cloneChecksum(Checksum checksum) { + if (checksum instanceof CRC64NVME) { + return (Checksum) ((CRC64NVME) checksum).clone(); + } + + throw new IllegalStateException("Unsupported checksum"); + } + + @Override + public byte[] getChecksumBytes() { + return longToByte(getChecksum().getValue()); + } +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java new file mode 100644 index 000000000000..2b4bc6a5c1f0 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java @@ -0,0 +1,53 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static software.amazon.awssdk.utils.NumericUtils.longToByte; + +import java.util.function.Function; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Class that provides functionality for combining CRC checksums with mark and reset capabilities. + * + *

+ * This class is intended for use with checksums that do not provide a cloneable method. It uses + * combine methods to handle mark and reset operations efficiently. + *

+ */ +@SdkInternalApi +public final class CrcCloneOnMarkChecksum extends BaseCrcChecksum { + + private final Function cloneFunction; + + public CrcCloneOnMarkChecksum(Checksum checksum, Function cloneFunction) { + super(checksum); + this.cloneFunction = cloneFunction; + } + + @Override + Checksum cloneChecksum(Checksum checksum) { + // Use the function to clone the checksum + return cloneFunction.apply(checksum); + } + + @Override + public byte[] getChecksumBytes() { + byte[] valueBytes = longToByte(getValue()); + return new byte[] { valueBytes[4], valueBytes[5], valueBytes[6], valueBytes[7] }; + } +} \ No newline at end of file diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java new file mode 100644 index 000000000000..a1a0f6d499a4 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java @@ -0,0 +1,133 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.Validate; + +/** + * Utility class that provides methods for combining CRC checksums using Galois Field arithmetic. + * This class allows combining two CRC values into a single CRC that represents the concatenated + * data, without recalculating the CRC from scratch. + *

+ * The implementation of CRC combination was taken from the zlib source code here: + * https://github.com/luvit/zlib/blob/master/crc32.c + *

+ */ +@SdkInternalApi +public final class CrcCombineChecksumUtil { + + public static final int CRC_SIZE = 32; + + private CrcCombineChecksumUtil() { + } + + /** + * Generates the combine matrices for CRC calculations. + * + * @param polynomial The CRC polynomial. + * @return A 2D array representing the combine matrices. + */ + public static long[][] generateCombineMatrices(long polynomial) { + long[][] combineMatrices = new long[CRC_SIZE][CRC_SIZE]; + initializeFirstMatrix(combineMatrices, polynomial); + deriveRemainingMatrices(combineMatrices); + return combineMatrices; + } + + /** + * Combines two CRC values into a single CRC using the specified combine matrices. + * + * The combination is performed using Galois Field arithmetic to effectively merge + * two CRC checksums that correspond to two separate data blocks. This method allows + * calculating the CRC for the concatenated data without having to recompute the CRC + * from scratch, which can significantly improve performance for large datasets. + *

+ * THIS COMBINE FUNCTION HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. + * The code comes from + * https://github.com/luvit/zlib/blob/master/crc32.c. + *

+ * @param crc1 The first CRC value. + * @param crc2 The second CRC value. + * @param originalLengthOfCrc2 The length of the original data for the second CRC. + * This represents the length of data used to compute {@code crc2}. + * @param combineMatrices The combine matrices used for combining CRCs. + * These matrices are precomputed to facilitate efficient combination. + * @return The combined CRC value representing the CRC for the concatenated data of both CRC values. + * @throws IllegalArgumentException if {@code originalLengthOfCrc2} is negative. + */ + public static long combine(long crc1, long crc2, long originalLengthOfCrc2, long[][] combineMatrices) { + Validate.isNotNegative(originalLengthOfCrc2, "The length of the original data for the " + + "second CRC value must be positive."); + if (originalLengthOfCrc2 == 0) { + return crc1; + } + int matrixIndex = 2; + while (originalLengthOfCrc2 != 0) { + ++matrixIndex; + if ((originalLengthOfCrc2 & 1) != 0) { + crc1 = gf2MatrixTimes(combineMatrices[matrixIndex], crc1); + } + originalLengthOfCrc2 >>= 1; + } + crc1 ^= crc2; + return crc1; + } + + /** + * Multiplies a Galois Field matrix with a vector. + * + * @param matrix The matrix to be multiplied. + * @param vector The vector to be multiplied. + * @return The result of the multiplication. + */ + private static long gf2MatrixTimes(long[] matrix, long vector) { + long sum = 0; + for (long l : matrix) { + if (vector == 0) { + break; + } + if ((vector & 1) != 0) { + sum ^= l; + } + vector >>= 1; + } + return sum; + } + + private static void initializeFirstMatrix(long[][] combineMatrices, long polynomial) { + combineMatrices[0][0] = polynomial; + long row = 1; + for (int i = 1; i < CRC_SIZE; i++) { + combineMatrices[0][i] = row; + row <<= 1; + } + } + + /** + * Derives the remaining matrices for the combination process. + * + * @param combineMatrices The combine matrices to be derived. + */ + private static void deriveRemainingMatrices(long[][] combineMatrices) { + for (int i = 0; i < CRC_SIZE - 1; i++) { + for (int j = 0; j < CRC_SIZE; j++) { + combineMatrices[i + 1][j] = gf2MatrixTimes(combineMatrices[i], combineMatrices[i][j]); + } + } + } + +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java new file mode 100644 index 000000000000..ff0f564527e6 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Functional interface for combining two CRC values along with the data length. + */ +@SdkInternalApi +@FunctionalInterface +public interface CrcCombineFunction { + /** + * Combines two CRC values along with the data length to produce the resulting combined checksum. + * + * @param crc1 The first CRC value. + * @param crc2 The second CRC value. + * @param length The length of the data. + * @return The combined CRC value. + */ + long combine(long crc1, long crc2, long length); +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java new file mode 100644 index 000000000000..c1a27dfec4e2 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static software.amazon.awssdk.utils.NumericUtils.longToByte; + +import java.util.function.BiFunction; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; + +/** + * Class that provides functionality for combining CRC checksums with mark and reset capabilities. + * + *

+ * This class allows combining two CRC values, making it efficient to update and restore the checksum + * during mark and reset operations. It is particularly useful in scenarios where a checksum needs to + * be marked, potentially reset to a previous state, and combined with additional data. + *

+ * + *

+ * The class maintains an internal {@link Checksum} instance and uses a {@link BiFunction} to combine + * two CRC values when necessary. The combine function is applied to a pair of CRC values, along with + * the length of the data, to produce the resulting combined checksum. + *

+ */ +@SdkInternalApi +public class CrcCombineOnMarkChecksum implements SdkChecksum { + private final Checksum crc; + private long dataLengthForChecksum = 0; + private Long crcAtMark; + private long markedDataLength = 0; + private boolean isResetDone = false; + + private final CrcCombineFunction crcCombineFunction; + + + public CrcCombineOnMarkChecksum(Checksum checksum, + CrcCombineFunction crcCombineFunction) { + this.crc = checksum; + this.crcCombineFunction = crcCombineFunction; + } + + @Override + public byte[] getChecksumBytes() { + byte[] valueBytes = longToByte(getValue()); + return new byte[] { valueBytes[4], valueBytes[5], valueBytes[6], valueBytes[7] }; + } + + @Override + public void mark(int readLimit) { + if (dataLengthForChecksum > 0) { + saveMarkState(); + } + } + + @Override + public void update(int b) { + crc.update(b); + dataLengthForChecksum += 1; + } + + @Override + public void update(byte[] b, int off, int len) { + crc.update(b, off, len); + dataLengthForChecksum += len; + } + + @Override + public long getValue() { + if (canRestoreMarkedState()) { + return crcCombineFunction.combine(crcAtMark, crc.getValue(), dataLengthForChecksum - markedDataLength); + } + return crc.getValue(); + } + + @Override + public void reset() { + if (crcAtMark != null) { + crc.reset(); + dataLengthForChecksum = markedDataLength; + } else { + crc.reset(); + dataLengthForChecksum = 0; + } + isResetDone = true; + } + + private void saveMarkState() { + crcAtMark = crc.getValue(); + markedDataLength = dataLengthForChecksum; + } + + private boolean canRestoreMarkedState() { + return crcAtMark != null && isResetDone; + } + +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/DigestAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java similarity index 96% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/DigestAlgorithm.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java index 28a2004ed7d4..cd5e63a070d3 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/DigestAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.util; +package software.amazon.awssdk.checksums.internal; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Md5Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java similarity index 93% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Md5Checksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java index e7ccabd2d32e..6b1a5d7cc67b 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Md5Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums.internal; import java.security.MessageDigest; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.auth.aws.internal.signer.util.DigestAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; /** * Implementation of {@link SdkChecksum} to calculate an MD5 checksum. diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32CChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java similarity index 97% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32CChecksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java index 6c1d07175d41..3b7a49456aa9 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32CChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java @@ -13,12 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums.internal; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; - /* * THIS FILE HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. * @@ -563,6 +562,9 @@ public final class SdkCrc32CChecksum implements Checksum, Cloneable { 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 }; + private static final long POLYNOMIAL = 0x82F63B78; + + private static final long[][] COMBINE_MATRICES = CrcCombineChecksumUtil.generateCombineMatrices(POLYNOMIAL); /** * the current CRC value, bit-flipped */ @@ -580,6 +582,19 @@ public static SdkCrc32CChecksum create() { return new SdkCrc32CChecksum(); } + /** + * Combines the CRCs of two parts.Please refer {@link CrcCombineChecksumUtil#combine(long, long, long, long[][])} + * + * + * @param crc1 The CRC of the first part. + * @param crc2 The CRC of the second part. + * @param originalLengthOfCrc2 The length of the second part's CRC before combining. + * @return The combined CRC. + */ + public static long combine(long crc1, long crc2 , long originalLengthOfCrc2) { + return CrcCombineChecksumUtil.combine(crc1, crc2, originalLengthOfCrc2, COMBINE_MATRICES); + } + @Override public long getValue() { long ret = crc; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java similarity index 97% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32Checksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java index 48f00aaac35e..7149695c16bf 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/SdkCrc32Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums.internal; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -554,6 +554,11 @@ public final class SdkCrc32Checksum implements Checksum, Cloneable { 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 }; + + private static final long POLYNOMIAL = 0xEDB88320L; + + private static final long[][] COMBINE_MATRICES = CrcCombineChecksumUtil.generateCombineMatrices(POLYNOMIAL); + /** * the current CRC value, bit-flipped */ @@ -573,6 +578,18 @@ public static SdkCrc32Checksum create() { return new SdkCrc32Checksum(); } + /** + * Combines two CRC32 values into a single CRC using the specified combine matrices. + * Please refer {@link CrcCombineChecksumUtil#combine(long, long, long, long[][])} + * + * @param crcPair A pair of CRC values to combine. + * @param originalLengthOfCrc32 The length of the original data. + * @return The combined CRC value. + */ + public static long combine(Long crcPairOne, Long crcPair, long originalLengthOfCrc32) { + return CrcCombineChecksumUtil.combine(crcPairOne, crcPair, originalLengthOfCrc32, COMBINE_MATRICES); + } + @Override public long getValue() { return ~crc & 0xffffffffL; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha1Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java similarity index 93% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha1Checksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java index 15852aceaf1e..d377a669f2c1 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha1Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums.internal; import java.security.MessageDigest; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.auth.aws.internal.signer.util.DigestAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; /** * Implementation of {@link SdkChecksum} to calculate an Sha-1 checksum. diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha256Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java similarity index 93% rename from core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha256Checksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java index b6fa6338e023..472375d74dfd 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Sha256Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java @@ -13,11 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; +package software.amazon.awssdk.checksums.internal; import java.security.MessageDigest; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.auth.aws.internal.signer.util.DigestAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; /** * Implementation of {@link SdkChecksum} to calculate an Sha-256 Checksum. diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java new file mode 100644 index 000000000000..04386fa95f80 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.lang.reflect.Constructor; +import java.util.Optional; +import software.amazon.awssdk.checksums.internal.ConstructorCache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConstructorCacheTest { + + private ConstructorCache constructorCache; + + @BeforeEach + void setUp() { + constructorCache = new ConstructorCache(); + } + + @Test + void testGetConstructor_existingClassWithNoArgsConstructor() { + // Test with a known class that has a no-args constructor, e.g., String + Optional> constructor = constructorCache.getConstructor("java.lang.String"); + assertTrue(constructor.isPresent()); + assertEquals(String.class, constructor.get().getDeclaringClass()); + } + + @Test + void testGetConstructor_existingClassWithoutNoArgsConstructor() { + // Test with a class that doesn't have a no-args constructor + Optional> constructor = constructorCache.getConstructor("java.util.Scanner"); + assertFalse(constructor.isPresent()); + } + + @Test + void testGetConstructor_nonExistingClass() { + // Test with a non-existing class name + Optional> constructor = constructorCache.getConstructor("non.existent.ClassName"); + assertFalse(constructor.isPresent()); + } +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java new file mode 100644 index 000000000000..5e9a5d1e54fd --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; + +public class CRTBasedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createCrtCrc32C(); + } +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java new file mode 100644 index 000000000000..9b408aa4ab75 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + + +import java.util.stream.Stream; +import org.junit.jupiter.params.provider.Arguments; + +abstract class Crc32CChecksumTest extends CrcChecksumBase { + + protected static Stream provideParametersForCrcCheckSumValues() { + return Stream.of( + Arguments.of("00000000000000000000000000000000a245d57d") + ); + } + + protected static Stream provideValidateEncodedBase64ForCrc() { + return Stream.of( + Arguments.of("Nks/tw==") + ); + } + + protected static Stream provideValidateMarkAndResetForCrc() { + return Stream.of( + Arguments.of("Nks/tw==") + ); + } + + protected static Stream provideValidateMarkForCrc() { + return Stream.of( + Arguments.of("crUfeA==") + ); + } + + protected static Stream provideValidateSingleMarksForCrc() { + return Stream.of( + Arguments.of("eNkvgQ==") + ); + } + + protected static Stream provideValidateMultipleMarksForCrc() { + return Stream.of( + Arguments.of("Bf1liQ==") + ); + } + + protected static Stream provideValidateResetWithoutMarkForCrc() { + return Stream.of( + Arguments.of("eNkvgQ==") + ); + } + +} \ No newline at end of file diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java new file mode 100644 index 000000000000..061b4f48f499 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java @@ -0,0 +1,71 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.provider.Arguments; + +class Crc32ChecksumTest extends CrcChecksumBase { + + @BeforeEach + public void setUp() { + sdkChecksum = new Crc32Checksum(); + } + + protected static Stream provideParametersForCrcCheckSumValues() { + return Stream.of( + Arguments.of("000000000000000000000000000000001fc2e6d2") + ); + } + + protected static Stream provideValidateEncodedBase64ForCrc() { + return Stream.of( + Arguments.of("NSRBwg==") + ); + } + + protected static Stream provideValidateMarkAndResetForCrc() { + return Stream.of( + Arguments.of("NSRBwg==") + ); + } + + protected static Stream provideValidateMarkForCrc() { + return Stream.of( + Arguments.of("i9aeUg==") + ); + } + + protected static Stream provideValidateSingleMarksForCrc() { + return Stream.of( + Arguments.of("0OA5ag==") + ); + } + + protected static Stream provideValidateMultipleMarksForCrc() { + return Stream.of( + Arguments.of("xSjLBA==") + ); + } + + protected static Stream provideValidateResetWithoutMarkForCrc() { + return Stream.of( + Arguments.of("0OA5ag==") + ); + } + +} \ No newline at end of file diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java new file mode 100644 index 000000000000..c9ad393bb092 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java @@ -0,0 +1,112 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.utils.BinaryUtils; + +abstract class CrcChecksumBase { + + protected SdkChecksum sdkChecksum; + + static final String TEST_STRING = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + @ParameterizedTest + @MethodSource("provideParametersForCrcCheckSumValues") + void crcCheckSumValues(String expectedChecksum) throws UnsupportedEncodingException { + byte[] bytes = TEST_STRING.getBytes("UTF-8"); + sdkChecksum.update(bytes, 0, bytes.length); + assertEquals(expectedChecksum, getAsString(sdkChecksum.getChecksumBytes())); + } + + @ParameterizedTest + @MethodSource("provideValidateEncodedBase64ForCrc") + void validateEncodedBase64ForCrc(String expectedChecksum) { + sdkChecksum.update("abc".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMarkAndResetForCrc") + void validateMarkAndResetForCrc(String expectedChecksum) { + sdkChecksum.update("ab".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("xyz".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("c".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMarkForCrc") + void validateMarkForCrc(String expectedChecksum) { + sdkChecksum.update("Hello ".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(4); + sdkChecksum.update("world".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateSingleMarksForCrc") + void validateSingleMarksForCrc(String expectedChecksum) { + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMultipleMarksForCrc") + void validateMultipleMarksForCrc(String expectedChecksum) { + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(5); + sdkChecksum.update("gamma".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("delta".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateResetWithoutMarkForCrc") + void validateResetWithoutMarkForCrc(String expectedChecksum) { + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + private String getAsString(byte[] checksumBytes) { + return String.format("%040x", new BigInteger(1, checksumBytes)); + } +} + + diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java new file mode 100644 index 000000000000..80810970e4d3 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +public class DefaultCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.create(); + } + + + @AfterEach + void tearDown() { + sdkChecksum = null; + // Try to force garbage collection to clear weak references + System.gc(); + System.runFinalization(); + } + +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java similarity index 83% rename from core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java rename to core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java index ac0a2b7791b7..82e768af16ed 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java @@ -13,17 +13,19 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.checksums; +package software.amazon.awssdk.checksums.internal; import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; public class DefaultChecksumAlgorithmTest { @Test public void hasCRC32C() { - assertEquals("CRC32C", DefaultChecksumAlgorithm.CRC32C.algorithmId()); + Assertions.assertEquals("CRC32C", DefaultChecksumAlgorithm.CRC32C.algorithmId()); } @Test diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java new file mode 100644 index 000000000000..1acc96dc167a --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.DisabledOnJre; +import org.junit.jupiter.api.condition.JRE; + +@DisabledOnJre(JRE.JAVA_8) +public class Java9BasedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createJavaCrc32C(); + } + +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java new file mode 100644 index 000000000000..98189d8bc28d --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; + +public class SdkImplmenetedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createSdkBasedCrc32C(); + } +} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java index 0cae6f6364eb..619a9f15ce3d 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java @@ -29,12 +29,12 @@ import java.util.Collections; import java.util.List; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.Header; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java index 23d4b0dc33d3..a0c373792eb7 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java @@ -33,11 +33,11 @@ import java.util.List; import org.reactivestreams.Publisher; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.Header; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4ChunkExtensionProvider; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java index e13b8726d15b..2cf8e2cfef26 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java @@ -29,10 +29,10 @@ import java.util.stream.Collectors; import org.reactivestreams.Publisher; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumSubscriber; import software.amazon.awssdk.utils.Validate; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/ConstantChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/ConstantChecksum.java index b3f33db111f3..76f78f399fb9 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/ConstantChecksum.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/ConstantChecksum.java @@ -17,6 +17,7 @@ import java.nio.charset.StandardCharsets; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; /** * Implementation of {@link SdkChecksum} to provide a constant checksum. diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32CChecksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32CChecksum.java deleted file mode 100644 index 8e0f94fd5ea1..000000000000 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32CChecksum.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.Checksum; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.crt.checksums.CRC32C; -import software.amazon.awssdk.utils.ClassLoaderHelper; - -/** - * Implementation of {@link SdkChecksum} to calculate an CRC32C checksum. - */ -@SdkInternalApi -public class Crc32CChecksum implements SdkChecksum { - - private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C"; - private static final ThreadLocal IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32CChecksum::isCrtAvailable); - - private Checksum crc32c; - private Checksum lastMarkedCrc32C; - - /** - * Creates CRT Based Crc32C checksum if Crt classpath for Crc32c is loaded, else create Sdk Implemented Crc32c - */ - public Crc32CChecksum() { - if (IS_CRT_AVAILABLE.get()) { - crc32c = new CRC32C(); - } else { - crc32c = SdkCrc32CChecksum.create(); - } - } - - private static boolean isCrtAvailable() { - try { - ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32C, false); - } catch (ClassNotFoundException e) { - return false; - } - - return true; - } - - private static byte[] longToByte(Long input) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(input); - return buffer.array(); - } - - @Override - public byte[] getChecksumBytes() { - return Arrays.copyOfRange(longToByte(crc32c.getValue()), 4, 8); - } - - @Override - public void mark(int readLimit) { - this.lastMarkedCrc32C = cloneChecksum(crc32c); - } - - @Override - public void update(int b) { - crc32c.update(b); - } - - @Override - public void update(byte[] b, int off, int len) { - crc32c.update(b, off, len); - } - - @Override - public long getValue() { - return crc32c.getValue(); - } - - @Override - public void reset() { - if (lastMarkedCrc32C == null) { - crc32c.reset(); - } else { - crc32c = cloneChecksum(lastMarkedCrc32C); - } - } - - private Checksum cloneChecksum(Checksum checksum) { - if (checksum instanceof CRC32C) { - return (Checksum) ((CRC32C) checksum).clone(); - } - - if (checksum instanceof SdkCrc32CChecksum) { - return (Checksum) ((SdkCrc32CChecksum) checksum).clone(); - } - - throw new IllegalStateException("Unsupported checksum"); - } -} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32Checksum.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32Checksum.java deleted file mode 100644 index 9c32f596994f..000000000000 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/checksums/Crc32Checksum.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.auth.aws.internal.signer.checksums; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.Checksum; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.crt.checksums.CRC32; -import software.amazon.awssdk.utils.ClassLoaderHelper; - -/** - * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum. - */ -@SdkInternalApi -public class Crc32Checksum implements SdkChecksum { - private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32"; - private static final ThreadLocal IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32Checksum::isCrtAvailable); - - private Checksum crc32; - private Checksum lastMarkedCrc32; - - - /** - * Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32. - */ - public Crc32Checksum() { - if (IS_CRT_AVAILABLE.get()) { - crc32 = new CRC32(); - } else { - crc32 = SdkCrc32Checksum.create(); - } - } - - private static boolean isCrtAvailable() { - try { - ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32, false); - } catch (ClassNotFoundException e) { - return false; - } - - return true; - } - - private static byte[] longToByte(Long input) { - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(input); - return buffer.array(); - } - - @Override - public byte[] getChecksumBytes() { - return Arrays.copyOfRange(longToByte(crc32.getValue()), 4, 8); - } - - @Override - public void mark(int readLimit) { - this.lastMarkedCrc32 = cloneChecksum(crc32); - } - - @Override - public void update(int b) { - crc32.update(b); - } - - @Override - public void update(byte[] b, int off, int len) { - crc32.update(b, off, len); - } - - @Override - public long getValue() { - return crc32.getValue(); - } - - @Override - public void reset() { - if (lastMarkedCrc32 == null) { - crc32.reset(); - } else { - crc32 = cloneChecksum(lastMarkedCrc32); - } - } - - private Checksum cloneChecksum(Checksum checksum) { - if (checksum instanceof CRC32) { - return (Checksum) ((CRC32) checksum).clone(); - } - - if (checksum instanceof SdkCrc32Checksum) { - return (Checksum) ((SdkCrc32Checksum) checksum).clone(); - } - - throw new IllegalStateException("Unsupported checksum"); - } -} diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java index b4a36bb0d931..bacdb50d975b 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProvider.java @@ -18,7 +18,7 @@ import java.util.Collections; import java.util.List; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.Pair; diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java index 7053d40d8b39..a8eae5fa5320 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtil.java @@ -15,41 +15,20 @@ package software.amazon.awssdk.http.auth.aws.internal.signer.util; -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5; -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; -import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; - import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.Locale; -import java.util.Map; -import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.ConstantChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32CChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Md5Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha1Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha256Checksum; import software.amazon.awssdk.utils.FunctionalUtils; -import software.amazon.awssdk.utils.ImmutableMap; @SdkInternalApi public final class ChecksumUtil { private static final String CONSTANT_CHECKSUM = "CONSTANT"; - private static final Map> CHECKSUM_MAP = ImmutableMap.of( - SHA256.algorithmId(), Sha256Checksum::new, - SHA1.algorithmId(), Sha1Checksum::new, - CRC32.algorithmId(), Crc32Checksum::new, - CRC32C.algorithmId(), Crc32CChecksum::new, - MD5.algorithmId(), Md5Checksum::new - ); - private ChecksumUtil() { } @@ -69,15 +48,15 @@ public static String checksumHeaderName(ChecksumAlgorithm checksumAlgorithm) { */ public static SdkChecksum fromChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { String algorithmId = checksumAlgorithm.algorithmId(); - Supplier checksumSupplier = CHECKSUM_MAP.get(algorithmId); - if (checksumSupplier != null) { - return checksumSupplier.get(); - } - if (CONSTANT_CHECKSUM.equals(algorithmId)) { return new ConstantChecksum(((ConstantChecksumAlgorithm) checksumAlgorithm).value); } + SdkChecksum checksum = SdkChecksum.forAlgorithm(checksumAlgorithm); + if (checksum != null) { + return checksum; + } + throw new UnsupportedOperationException("Checksum not supported for " + algorithmId); } @@ -93,6 +72,12 @@ public static void readAll(InputStream inputStream) { }); } + public static byte[] longToByte(Long input) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(input); + return buffer.array(); + } + /** * An implementation of a {@link ChecksumAlgorithm} that will map to {@link ConstantChecksum}, which provides a constant * checksum. This isn't super useful, but is needed in cases such as signing, where the content-hash (a diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/OptionalDependencyLoaderUtil.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/OptionalDependencyLoaderUtil.java index b0acc3b039aa..9bdb289dcc3c 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/OptionalDependencyLoaderUtil.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/OptionalDependencyLoaderUtil.java @@ -15,13 +15,14 @@ package software.amazon.awssdk.http.auth.aws.internal.signer.util; +import static software.amazon.awssdk.utils.DependencyValidate.requireClass; + import java.time.Clock; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.http.auth.aws.crt.internal.signer.DefaultAwsCrtV4aHttpSigner; import software.amazon.awssdk.http.auth.aws.eventstream.internal.signer.EventStreamV4PayloadSigner; import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; -import software.amazon.awssdk.utils.ClassLoaderHelper; import software.amazon.awssdk.utils.Logger; /** @@ -42,24 +43,6 @@ public final class OptionalDependencyLoaderUtil { private OptionalDependencyLoaderUtil() { } - /** - * A helpful method that checks that some class is available on the class-path. If it fails to load, it will throw an - * exception based on why it failed to load. This should be used in cases where certain dependencies are optional, but the - * dependency is used at compile-time for strong typing (i.e. {@link EventStreamV4PayloadSigner}). - */ - private static void requireClass(String classPath, String module, String feature) { - try { - ClassLoaderHelper.loadClass(classPath, false); - } catch (ClassNotFoundException e) { - LOG.debug(() -> "Cannot find the " + classPath + " class: ", e); - String msg = String.format("Could not load class. You must add a dependency on the '%s' module to enable the %s " - + "feature: ", module, feature); - throw new RuntimeException(msg, e); - } catch (Exception e) { - throw new RuntimeException(String.format("Could not load class (%s): ", classPath), e); - } - } - public static DefaultAwsCrtV4aHttpSigner getDefaultAwsCrtV4aHttpSigner() { requireClass(HTTP_AUTH_AWS_CRT_PATH, HTTP_AUTH_AWS_CRT_MODULE, "CRT-V4a signing"); return new DefaultAwsCrtV4aHttpSigner(); diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java index c61523f49a5c..186cb21f9d66 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java @@ -30,6 +30,7 @@ import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.Header; import software.amazon.awssdk.http.SdkHttpRequest; diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java index cfb984f81c28..25417a1acb81 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummerTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; import static software.amazon.awssdk.http.auth.aws.internal.signer.FlexibleChecksummer.option; @@ -91,14 +92,16 @@ public void checksummerAsync_withOneChecksum_shouldAddOneChecksum() { } @Test - public void checksummer_withTwoChecksums_shouldAddTwoChecksums() { + public void checksummer_withMultipleChecksums_shouldAddAllChecksums() { FlexibleChecksummer checksummer = new FlexibleChecksummer( option().headerName("sha256").algorithm(SHA256).formatter(BinaryUtils::toHex).build(), - option().headerName("crc32").algorithm(CRC32).formatter(BinaryUtils::toBase64).build() + option().headerName("crc32").algorithm(CRC32).formatter(BinaryUtils::toBase64).build(), + option().headerName("crc64nvme").algorithm(CRC64NVME).formatter(BinaryUtils::toBase64).build() ); SdkHttpRequest expectedRequest = request .putHeader("sha256", "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") .putHeader("crc32", "jHNlIQ==") + .putHeader("crc64nvme", "5O33DmauDQI=") .build(); checksummer.checksum(payload, request); @@ -107,14 +110,16 @@ public void checksummer_withTwoChecksums_shouldAddTwoChecksums() { } @Test - public void checksummerAsync_withTwoChecksums_shouldAddTwoChecksums() { + public void checksummerAsync_withMultipleChecksums_shouldAddAllChecksums() { FlexibleChecksummer checksummer = new FlexibleChecksummer( option().headerName("sha256").algorithm(SHA256).formatter(BinaryUtils::toBase64).build(), - option().headerName("crc32").algorithm(CRC32).formatter(BinaryUtils::toHex).build() + option().headerName("crc32").algorithm(CRC32).formatter(BinaryUtils::toHex).build(), + option().headerName("crc64nvme").algorithm(CRC64NVME).formatter(BinaryUtils::toBase64).build() ); SdkHttpRequest expectedRequest = request .putHeader("sha256", "LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564=") .putHeader("crc32", "8c736521") + .putHeader("crc64nvme", "5O33DmauDQI=") .build(); checksummer.checksum(payloadAsync, request); diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java index 1aa9157213d3..b2a09996eccd 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java @@ -23,9 +23,10 @@ import java.util.Arrays; import java.util.Collections; import org.junit.jupiter.api.Test; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.Crc32Checksum; +import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.internal.Sha256Checksum; import software.amazon.awssdk.utils.BinaryUtils; public class ChecksumInputStreamTest { @@ -50,17 +51,21 @@ public void read_withMultipleChecksums_shouldComputeCorrectChecksums() { String testString = "AWS SDK for Java"; String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; String expectedCrc32Digest = "4ac37ece"; + String expectedCrc64Digest = "7c05fe704e3e02bc"; ByteArrayInputStream backingStream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); SdkChecksum sha256Checksum = new Sha256Checksum(); SdkChecksum crc32Checksum = new Crc32Checksum(); - ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Arrays.asList(sha256Checksum, crc32Checksum)); + SdkChecksum crc64NvmeChecksum = new Crc64NvmeChecksum(); + ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Arrays.asList(sha256Checksum, crc32Checksum, crc64NvmeChecksum)); readAll(inputStream); String computedSha256Digest = BinaryUtils.toHex(sha256Checksum.getChecksumBytes()); String computedCrc32Digest = BinaryUtils.toHex(crc32Checksum.getChecksumBytes()); + String computedCrc64NvmeDigest = BinaryUtils.toHex(crc64NvmeChecksum.getChecksumBytes()); assertThat(computedSha256Digest).isEqualTo(expectedSha256Digest); assertThat(computedCrc32Digest).isEqualTo(expectedCrc32Digest); + assertThat(computedCrc64NvmeDigest).isEqualTo(expectedCrc64Digest); } } diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java index 35ebf85e03f3..c6a4be5fcecb 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java @@ -19,7 +19,7 @@ import java.util.Collections; import org.reactivestreams.Subscriber; import org.reactivestreams.tck.TestEnvironment; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.Sha256Checksum; /** * TCK verifiation test for {@link ChecksumSubscriber}. diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java index 16b5b6bed9f7..712e8a752698 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java @@ -30,9 +30,10 @@ import java.util.Collections; import org.junit.jupiter.api.Test; import org.reactivestreams.Subscription; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.SdkChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.Crc32Checksum; +import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.internal.Sha256Checksum; import software.amazon.awssdk.utils.BinaryUtils; public class ChecksumSubscriberTest { @@ -58,19 +59,23 @@ public void checksum_withMultipleChecksums_shouldComputeCorrectChecksums() { String testString = "AWS SDK for Java"; String expectedSha256Digest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; String expectedCrc32Digest = "4ac37ece"; + String expectedCrc64Digest = "7c05fe704e3e02bc"; SdkChecksum sha256Checksum = new Sha256Checksum(); SdkChecksum crc32Checksum = new Crc32Checksum(); - ChecksumSubscriber subscriber = new ChecksumSubscriber(Arrays.asList(sha256Checksum, crc32Checksum)); + SdkChecksum crc64NvmeChecksum = new Crc64NvmeChecksum(); + ChecksumSubscriber subscriber = new ChecksumSubscriber(Arrays.asList(sha256Checksum, crc32Checksum, crc64NvmeChecksum)); Flowable publisher = Flowable.just(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); publisher.subscribe(subscriber); joinLikeSync(subscriber.completeFuture()); String computedSha256Digest = BinaryUtils.toHex(sha256Checksum.getChecksumBytes()); String computedCrc32Digest = BinaryUtils.toHex(crc32Checksum.getChecksumBytes()); + String computedCrc64NvmeDigest = BinaryUtils.toHex(crc64NvmeChecksum.getChecksumBytes()); assertThat(computedSha256Digest).isEqualTo(expectedSha256Digest); assertThat(computedCrc32Digest).isEqualTo(expectedCrc32Digest); + assertThat(computedCrc64NvmeDigest).isEqualTo(expectedCrc64Digest); } @Test diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java index 06e3b1d4839a..f3eb940f1bf3 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java @@ -17,9 +17,11 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; @@ -33,11 +35,12 @@ import java.io.UncheckedIOException; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32CChecksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Crc32Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Md5Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha1Checksum; -import software.amazon.awssdk.http.auth.aws.internal.signer.checksums.Sha256Checksum; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.internal.Crc32Checksum; +import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; +import software.amazon.awssdk.checksums.internal.Md5Checksum; +import software.amazon.awssdk.checksums.internal.Sha1Checksum; +import software.amazon.awssdk.checksums.internal.Sha256Checksum; public class ChecksumUtilTest { @@ -48,6 +51,7 @@ public void checksumHeaderName_shouldFormatName() { assertEquals("x-amz-checksum-crc32", checksumHeaderName(CRC32)); assertEquals("x-amz-checksum-crc32c", checksumHeaderName(CRC32C)); assertEquals("x-amz-checksum-md5", checksumHeaderName(MD5)); + assertEquals("x-amz-checksum-crc64nvme", checksumHeaderName(CRC64NVME)); } @Test @@ -55,8 +59,10 @@ public void fromChecksumAlgorithm_mapsToCorrectSdkChecksum() { assertEquals(Sha256Checksum.class, fromChecksumAlgorithm(SHA256).getClass()); assertEquals(Sha1Checksum.class, fromChecksumAlgorithm(SHA1).getClass()); assertEquals(Crc32Checksum.class, fromChecksumAlgorithm(CRC32).getClass()); - assertEquals(Crc32CChecksum.class, fromChecksumAlgorithm(CRC32C).getClass()); + assertInstanceOf(SdkChecksum.class, fromChecksumAlgorithm(CRC32C)); assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); + assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); + assertEquals(Crc64NvmeChecksum.class, fromChecksumAlgorithm(CRC64NVME).getClass()); } @Test diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Algorithm.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Algorithm.java index 46813da33da5..8054257580ff 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Algorithm.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Algorithm.java @@ -22,7 +22,10 @@ /** * Enum that indicates all the checksums supported by Flexible checksums in a Service Request/Response Header. + * + * @deprecated this class is deprecated and will not be updated. */ +@Deprecated @SdkPublicApi public enum Algorithm { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32CChecksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32CChecksum.java index 2b0e69c9b6ea..e8b184747b3f 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32CChecksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32CChecksum.java @@ -26,7 +26,9 @@ /** * Implementation of {@link SdkChecksum} to calculate an CRC32C checksum. + * @deprecated this class is deprecated and subject to removal. */ +@Deprecated @SdkInternalApi public class Crc32CChecksum implements SdkChecksum { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32Checksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32Checksum.java index ed6afbcba598..e0c6f9237f8d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32Checksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Crc32Checksum.java @@ -26,7 +26,9 @@ /** * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum. + * @deprecated this class is deprecated and subject to removal. */ +@Deprecated @SdkInternalApi public class Crc32Checksum implements SdkChecksum { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Md5Checksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Md5Checksum.java index 79121df4fe9d..f690adb70440 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Md5Checksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Md5Checksum.java @@ -20,7 +20,9 @@ /** * Implementation of {@link SdkChecksum} to calculate an MD5 checksum. + * @deprecated this class is deprecated and subject to removal. */ +@Deprecated @SdkInternalApi public class Md5Checksum implements SdkChecksum { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/SdkChecksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/SdkChecksum.java index 871455bafa3e..574e55d51a4d 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/SdkChecksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/SdkChecksum.java @@ -22,7 +22,9 @@ /** * Extension of {@link Checksum} to support checksums and checksum validations used by the SDK that * are not provided by the JDK. + * @deprecated this class is deprecated and will not be updated. */ +@Deprecated @SdkPublicApi public interface SdkChecksum extends Checksum { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha1Checksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha1Checksum.java index 73542f0a9b90..962796836d02 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha1Checksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha1Checksum.java @@ -20,7 +20,9 @@ /** * Implementation of {@link SdkChecksum} to calculate an Sha-1 checksum. + * @deprecated this class is deprecated and subject to removal. */ +@Deprecated @SdkInternalApi public class Sha1Checksum implements SdkChecksum { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha256Checksum.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha256Checksum.java index 065f293a9770..b002cd0ad777 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha256Checksum.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/checksums/Sha256Checksum.java @@ -20,7 +20,9 @@ /** * Implementation of {@link SdkChecksum} to calculate an Sha-256 Checksum. + * @deprecated this class is deprecated and subject to removal. */ +@Deprecated @SdkInternalApi public class Sha256Checksum implements SdkChecksum { diff --git a/pom.xml b/pom.xml index f6a0b590bf97..1f2414dc2406 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ 2.2.21 1.17.1 1.37 - 0.31.1 + 0.31.3 5.10.0 diff --git a/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java b/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java index d5907cd21374..a9b7735260fa 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java @@ -120,7 +120,7 @@ public static Class loadClass(String fqcn, boolean classesFirst, * Attempt to get the current thread's class loader and fallback to the system classloader if null * @return a {@link ClassLoader} or null if none found */ - private static ClassLoader contextClassLoader() { + public static ClassLoader contextClassLoader() { ClassLoader threadClassLoader = Thread.currentThread().getContextClassLoader(); if (threadClassLoader != null) { return threadClassLoader; diff --git a/utils/src/main/java/software/amazon/awssdk/utils/DependencyValidate.java b/utils/src/main/java/software/amazon/awssdk/utils/DependencyValidate.java new file mode 100644 index 000000000000..98d6727a0f21 --- /dev/null +++ b/utils/src/main/java/software/amazon/awssdk/utils/DependencyValidate.java @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.utils; + +import software.amazon.awssdk.annotations.SdkProtectedApi; + +/** + * Utilities for loading of classes and objects which have optional dependencies, and therefore need to be safely checked at + * runtime in order to use. + */ +@SdkProtectedApi +public final class DependencyValidate { + private static final Logger LOG = Logger.loggerFor(DependencyValidate.class); + + private DependencyValidate() { + } + + /** + * A helpful method that checks that some class is available on the class-path. If it fails to load, it will throw an + * exception based on why it failed to load. This should be used in cases where certain dependencies are optional, but the + * dependency is used at compile-time for strong typing + */ + public static void requireClass(String classPath, String module, String feature) { + try { + ClassLoaderHelper.loadClass(classPath, false); + } catch (ClassNotFoundException e) { + LOG.debug(() -> "Cannot find the " + classPath + " class: ", e); + String msg = String.format("Could not load class. You must add a dependency on the '%s' module to enable the %s " + + "feature: ", module, feature); + throw new RuntimeException(msg, e); + } catch (Exception e) { + throw new RuntimeException(String.format("Could not load class (%s): ", classPath), e); + } + } +} diff --git a/utils/src/main/java/software/amazon/awssdk/utils/NumericUtils.java b/utils/src/main/java/software/amazon/awssdk/utils/NumericUtils.java index 47aab28f2b7a..1f7a38defb54 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/NumericUtils.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/NumericUtils.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.utils; +import java.nio.ByteBuffer; import java.time.Duration; import software.amazon.awssdk.annotations.SdkProtectedApi; @@ -50,4 +51,13 @@ public static Duration max(Duration a, Duration b) { return (a.compareTo(b) > 0) ? a : b; } + /** + * Converts a long to a byte array + */ + public static byte[] longToByte(Long input) { + Validate.paramNotNull(input, "input"); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(input); + return buffer.array(); + } } diff --git a/utils/src/test/java/software/amazon/awssdk/utils/NumericUtilsTest.java b/utils/src/test/java/software/amazon/awssdk/utils/NumericUtilsTest.java index 20fdaae3e5c2..65c74ae61367 100644 --- a/utils/src/test/java/software/amazon/awssdk/utils/NumericUtilsTest.java +++ b/utils/src/test/java/software/amazon/awssdk/utils/NumericUtilsTest.java @@ -15,8 +15,7 @@ package software.amazon.awssdk.utils; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.utils.NumericUtils.max; import static software.amazon.awssdk.utils.NumericUtils.min; @@ -34,51 +33,58 @@ public class NumericUtilsTest { @Test public void minTestDifferentDurations() { - assertThat(min(SHORT_DURATION, LONG_DURATION), is(SHORT_DURATION)); + assertThat(min(SHORT_DURATION, LONG_DURATION)).isEqualTo(SHORT_DURATION); } @Test public void minTestDifferentDurationsReverse() { - assertThat(min(LONG_DURATION, SHORT_DURATION), is(SHORT_DURATION)); + assertThat(min(LONG_DURATION, SHORT_DURATION)).isEqualTo(SHORT_DURATION); } @Test public void minTestSameDurations() { - assertThat(min(SHORT_DURATION, SHORT_SAME_DURATION), is(SHORT_SAME_DURATION)); + assertThat(min(SHORT_DURATION, SHORT_SAME_DURATION)).isEqualTo(SHORT_SAME_DURATION); } @Test public void minTestDifferentNegativeDurations() { - assertThat(min(NEGATIVE_SHORT_DURATION, NEGATIVE_LONG_DURATION), is(NEGATIVE_LONG_DURATION)); + assertThat(min(NEGATIVE_SHORT_DURATION, NEGATIVE_LONG_DURATION)).isEqualTo(NEGATIVE_LONG_DURATION); } @Test public void minTestNegativeSameDurations() { - assertThat(min(NEGATIVE_SHORT_DURATION, NEGATIVE_SHORT_SAME_DURATION), is(NEGATIVE_SHORT_DURATION)); + assertThat(min(NEGATIVE_SHORT_DURATION, NEGATIVE_SHORT_SAME_DURATION)).isEqualTo(NEGATIVE_SHORT_DURATION); } @Test public void maxTestDifferentDurations() { - assertThat(max(LONG_DURATION, SHORT_DURATION), is(LONG_DURATION)); + assertThat(max(LONG_DURATION, SHORT_DURATION)).isEqualTo(LONG_DURATION); } @Test public void maxTestDifferentDurationsReverse() { - assertThat(max(SHORT_DURATION, LONG_DURATION), is(LONG_DURATION)); + assertThat(max(SHORT_DURATION, LONG_DURATION)).isEqualTo(LONG_DURATION); } @Test public void maxTestSameDurations() { - assertThat(max(SHORT_DURATION, SHORT_SAME_DURATION), is(SHORT_SAME_DURATION)); + assertThat(max(SHORT_DURATION, SHORT_SAME_DURATION)).isEqualTo(SHORT_SAME_DURATION); } @Test public void maxTestDifferentNegativeDurations() { - assertThat(max(NEGATIVE_SHORT_DURATION, NEGATIVE_LONG_DURATION), is(NEGATIVE_SHORT_DURATION)); + assertThat(max(NEGATIVE_SHORT_DURATION, NEGATIVE_LONG_DURATION)).isEqualTo(NEGATIVE_SHORT_DURATION); } @Test public void maxTestNegativeSameDurations() { - assertThat(max(NEGATIVE_SHORT_DURATION, NEGATIVE_SHORT_SAME_DURATION), is(NEGATIVE_SHORT_DURATION)); + assertThat(max(NEGATIVE_SHORT_DURATION, NEGATIVE_SHORT_SAME_DURATION)).isEqualTo(NEGATIVE_SHORT_DURATION); + } + + @Test + public void longToByte() { + long input = 12345678l; + byte[] bytes = NumericUtils.longToByte(input); + assertThat(bytes).encodedAsBase64().isEqualTo(BinaryUtils.toBase64(bytes)); } } \ No newline at end of file