From 820ed1654d4448a69a808e07be5aa26915215792 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Mon, 14 Oct 2024 15:42:47 -0700 Subject: [PATCH] Enhancements in CRC32 and CRC32C Checksum implementations --- .../feature-AWSSDKforJavav2-3676574.json | 6 + .../amazon/awssdk/checksums/SdkChecksum.java | 4 +- .../checksums/internal/ConstructorCache.java | 83 +++++++++++ .../checksums/internal/Crc32CChecksum.java | 78 ---------- .../checksums/internal/Crc32Checksum.java | 66 ++++----- .../checksums/internal/Crc32cProvider.java | 86 +++++++++++ .../internal/CrcCloneOnMarkChecksum.java | 53 +++++++ .../internal/CrcCombineChecksumUtil.java | 133 ++++++++++++++++++ .../internal/CrcCombineFunction.java | 35 +++++ .../internal/CrcCombineOnMarkChecksum.java | 111 +++++++++++++++ .../checksums/internal/SdkCrc32CChecksum.java | 17 ++- .../checksums/internal/SdkCrc32Checksum.java | 17 +++ .../checksums/ConstructorCacheTest.java | 58 ++++++++ .../internal/CRTBasedCRC32CChecksumTest.java | 26 ++++ .../internal/Crc32CChecksumTest.java | 66 +++++++++ .../checksums/internal/Crc32ChecksumTest.java | 71 ++++++++++ .../checksums/internal/CrcChecksumBase.java | 112 +++++++++++++++ .../internal/DefaultCRC32CChecksumTest.java | 37 +++++ .../DefaultChecksumAlgorithmTest.java | 6 +- .../Java9BasedCRC32CChecksumTest.java | 30 ++++ .../SdkImplmenetedCRC32CChecksumTest.java | 26 ++++ .../signer/util/ChecksumUtilTest.java | 5 +- .../awssdk/utils/ClassLoaderHelper.java | 2 +- 23 files changed, 1002 insertions(+), 126 deletions(-) create mode 100644 .changes/next-release/feature-AWSSDKforJavav2-3676574.json create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java delete mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32CChecksum.java create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java create mode 100644 core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java rename core/checksums/src/test/java/software/amazon/awssdk/checksums/{ => internal}/DefaultChecksumAlgorithmTest.java (83%) create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java create mode 100644 core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java diff --git a/.changes/next-release/feature-AWSSDKforJavav2-3676574.json b/.changes/next-release/feature-AWSSDKforJavav2-3676574.json new file mode 100644 index 000000000000..ac072611aa20 --- /dev/null +++ b/.changes/next-release/feature-AWSSDKforJavav2-3676574.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS SDK for Java v2", + "contributor": "", + "description": "The SDK now defaults to Java built-in CRC32 and CRC32C(if it's Java 9+) implementations, resulting in improved performance." +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java index 996de1966a19..12d5025f82ea 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java @@ -18,8 +18,8 @@ import java.nio.ByteBuffer; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkProtectedApi; -import software.amazon.awssdk.checksums.internal.Crc32CChecksum; import software.amazon.awssdk.checksums.internal.Crc32Checksum; +import software.amazon.awssdk.checksums.internal.Crc32cProvider; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; import software.amazon.awssdk.checksums.internal.Md5Checksum; import software.amazon.awssdk.checksums.internal.Sha1Checksum; @@ -39,7 +39,7 @@ public interface SdkChecksum extends Checksum { static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) { switch (algorithm.algorithmId()) { case "CRC32C": - return new Crc32CChecksum(); + return Crc32cProvider.create(); case "CRC32": return new Crc32Checksum(); case "SHA1": diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java new file mode 100644 index 000000000000..a710ed0a0ef4 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java @@ -0,0 +1,83 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.lang.ref.WeakReference; +import java.lang.reflect.Constructor; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.ClassLoaderHelper; +import software.amazon.awssdk.utils.Logger; + +/** + * A cache that stores classes and their constructors by class name and class loader. + *

+ * This cache uses weak references to both class loaders and classes, allowing them to be garbage collected + * when no longer needed. It provides methods to retrieve the zero-argument constructor for a class, + * based on the current thread's context class loader or the system class loader. + *

+ * If a class or its zero-argument constructor cannot be found, an empty result is returned. + */ +@SdkInternalApi +public final class ConstructorCache { + private static final Logger log = Logger.loggerFor(ConstructorCache.class); + + /** + * Cache storing classes by class name and class loader. + * Uses weak references to allow garbage collection when not needed. + */ + private final Map>>>> classesByClassName = + new ConcurrentHashMap<>(); + + /** + * Retrieve the class for the given class name from the context or system class loader. + * Returns an empty result if the class is not found. + */ + private Optional> getClass(String className) { + Map>>> classesByClassLoader = + classesByClassName.computeIfAbsent(className, k -> Collections.synchronizedMap(new WeakHashMap<>())); + + ClassLoader classLoader = ClassLoaderHelper.contextClassLoader(); + Optional>> classRef = classesByClassLoader.computeIfAbsent(classLoader, k -> { + try { + Class clazz = classLoader.loadClass(className); + return Optional.of(new WeakReference<>(clazz)); + } catch (ClassNotFoundException e) { + return Optional.empty(); + } + }); + return classRef.map(WeakReference::get); + } + + /** + * Retrieve the zero-argument constructor for the given class name. + * Returns an empty result if no such constructor is found. + */ + public Optional> getConstructor(String className) { + return getClass(className).flatMap(clazz -> { + try { + return Optional.of(clazz.getConstructor()); + } catch (NoSuchMethodException e) { + log.debug(() -> "Classloader contains " + className + ", but without a zero-arg constructor.", e); + return Optional.empty(); + } + }); + } +} \ No newline at end of file diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32CChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32CChecksum.java deleted file mode 100644 index 9aa1cfe354e2..000000000000 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32CChecksum.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.checksums.internal; - -import static software.amazon.awssdk.utils.NumericUtils.longToByte; - -import java.util.Arrays; -import java.util.zip.Checksum; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.checksums.SdkChecksum; -import software.amazon.awssdk.crt.checksums.CRC32C; -import software.amazon.awssdk.utils.ClassLoaderHelper; - -/** - * Implementation of {@link SdkChecksum} to calculate an CRC32C checksum. - */ -@SdkInternalApi -public class Crc32CChecksum extends BaseCrcChecksum { - - private static final String CRT_CLASSPATH_FOR_CRC32C = "software.amazon.awssdk.crt.checksums.CRC32C"; - private static final ThreadLocal IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32CChecksum::isCrtAvailable); - - /** - * Creates CRT Based Crc32C checksum if Crt classpath for Crc32c is loaded, else create Sdk Implemented Crc32c - */ - public Crc32CChecksum() { - super(createChecksum()); - } - - private static Checksum createChecksum() { - if (IS_CRT_AVAILABLE.get()) { - return new CRC32C(); - } - // TODO: use Java implementation if it's Java 9+ - return SdkCrc32CChecksum.create(); - } - - private static boolean isCrtAvailable() { - try { - ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32C, false); - } catch (ClassNotFoundException e) { - return false; - } - - return true; - } - - @Override - public byte[] getChecksumBytes() { - return Arrays.copyOfRange(longToByte(getChecksum().getValue()), 4, 8); - } - - @Override - public Checksum cloneChecksum(Checksum checksum) { - if (checksum instanceof CRC32C) { - return (Checksum) ((CRC32C) checksum).clone(); - } - - if (checksum instanceof SdkCrc32CChecksum) { - return (Checksum) ((SdkCrc32CChecksum) checksum).clone(); - } - - throw new IllegalStateException("Unsupported checksum"); - } -} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java index b067e3c4f908..ac93a3e7f4e1 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java @@ -15,64 +15,50 @@ package software.amazon.awssdk.checksums.internal; -import static software.amazon.awssdk.utils.NumericUtils.longToByte; -import java.util.Arrays; -import java.util.zip.Checksum; +import java.util.zip.CRC32; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.checksums.SdkChecksum; -import software.amazon.awssdk.crt.checksums.CRC32; -import software.amazon.awssdk.utils.ClassLoaderHelper; -/** - * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum. - */ @SdkInternalApi -public class Crc32Checksum extends BaseCrcChecksum { - private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32"; - private static final ThreadLocal IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32Checksum::isCrtAvailable); +public final class Crc32Checksum implements SdkChecksum { + private final CrcCombineOnMarkChecksum crc32; - /** - * Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32. - */ public Crc32Checksum() { - super(createChecksum()); + // Delegates to CrcCombineOnMarkChecksum with CRC32 + this.crc32 = new CrcCombineOnMarkChecksum( + new CRC32(), + SdkCrc32Checksum::combine + ); } - private static Checksum createChecksum() { - if (IS_CRT_AVAILABLE.get()) { - return new CRC32(); - } - - // TODO: use Java implementation - return SdkCrc32Checksum.create(); + @Override + public byte[] getChecksumBytes() { + return crc32.getChecksumBytes(); } - private static boolean isCrtAvailable() { - try { - ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32, false); - } catch (ClassNotFoundException e) { - return false; - } - - return true; + @Override + public void mark(int readLimit) { + crc32.mark(readLimit); } @Override - public byte[] getChecksumBytes() { - return Arrays.copyOfRange(longToByte(getChecksum().getValue()), 4, 8); + public void update(int b) { + crc32.update(b); } @Override - public Checksum cloneChecksum(Checksum checksum) { - if (checksum instanceof CRC32) { - return (Checksum) ((CRC32) checksum).clone(); - } + public void update(byte[] b, int off, int len) { + crc32.update(b, off, len); + } - if (checksum instanceof SdkCrc32Checksum) { - return (Checksum) ((SdkCrc32Checksum) checksum).clone(); - } + @Override + public long getValue() { + return crc32.getValue(); + } - throw new IllegalStateException("Unsupported checksum"); + @Override + public void reset() { + crc32.reset(); } } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java new file mode 100644 index 000000000000..9b35f043a0f3 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32cProvider.java @@ -0,0 +1,86 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.crt.checksums.CRC32C; + +/** + * Utility class to provide different implementations of CRC32C checksum. This class supports the use of: 1. Java-based CRC32C + * (Java 9+ when available) 2. CRT-based CRC32C (when available) 3. SDK-based CRC32C (as fallback) + */ +@SdkInternalApi +public final class Crc32cProvider { + + + // Class paths for different CRC32C implementations + private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C"; + private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C"; + private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache(); + + // Private constructor to prevent instantiation + private Crc32cProvider() { + } + + /** + * Creates an instance of the SDK-based CRC32C checksum as a fallback. + * + * @return An SdkChecksum instance. + */ + static SdkChecksum createSdkBasedCrc32C() { + SdkCrc32CChecksum sdkChecksum = SdkCrc32CChecksum.create(); + return new CrcCloneOnMarkChecksum(sdkChecksum, checksumToClone -> ((SdkCrc32CChecksum) checksumToClone).clone()); + } + + /** + * Tries to create a Java 9-based CRC32C checksum. + * If it's not available, it tries to create a CRT-based checksum. + * If both are not available, it falls back to an SDK-based CRC32C checksum. + * + * @return An instance of {@link SdkChecksum}, based on the first available option. + */ + public static SdkChecksum create() { + SdkChecksum checksum = createJavaCrc32C(); + if (checksum == null) { + checksum = createCrtCrc32C(); + } + return checksum != null ? checksum : createSdkBasedCrc32C(); + } + + static SdkChecksum createCrtCrc32C() { + return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> { + try { + return new CrcCloneOnMarkChecksum((Checksum) constructor.newInstance(), checksumToClone -> + (Checksum) ((CRC32C) checksumToClone).clone()); + } catch (ClassCastException | ReflectiveOperationException e) { + throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e); + } + }).orElse(null); + } + + static SdkChecksum createJavaCrc32C() { + return CONSTRUCTOR_CACHE.getConstructor(JAVA_CRC32C_CLASS_PATH).map(constructor -> { + try { + return new CrcCombineOnMarkChecksum((Checksum) constructor.newInstance(), SdkCrc32CChecksum::combine); + } catch (ClassCastException | ReflectiveOperationException e) { + throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e); + } + }).orElse(null); + } + +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java new file mode 100644 index 000000000000..2b4bc6a5c1f0 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCloneOnMarkChecksum.java @@ -0,0 +1,53 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static software.amazon.awssdk.utils.NumericUtils.longToByte; + +import java.util.function.Function; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Class that provides functionality for combining CRC checksums with mark and reset capabilities. + * + *

+ * This class is intended for use with checksums that do not provide a cloneable method. It uses + * combine methods to handle mark and reset operations efficiently. + *

+ */ +@SdkInternalApi +public final class CrcCloneOnMarkChecksum extends BaseCrcChecksum { + + private final Function cloneFunction; + + public CrcCloneOnMarkChecksum(Checksum checksum, Function cloneFunction) { + super(checksum); + this.cloneFunction = cloneFunction; + } + + @Override + Checksum cloneChecksum(Checksum checksum) { + // Use the function to clone the checksum + return cloneFunction.apply(checksum); + } + + @Override + public byte[] getChecksumBytes() { + byte[] valueBytes = longToByte(getValue()); + return new byte[] { valueBytes[4], valueBytes[5], valueBytes[6], valueBytes[7] }; + } +} \ No newline at end of file diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java new file mode 100644 index 000000000000..a1a0f6d499a4 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineChecksumUtil.java @@ -0,0 +1,133 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.utils.Validate; + +/** + * Utility class that provides methods for combining CRC checksums using Galois Field arithmetic. + * This class allows combining two CRC values into a single CRC that represents the concatenated + * data, without recalculating the CRC from scratch. + *

+ * The implementation of CRC combination was taken from the zlib source code here: + * https://github.com/luvit/zlib/blob/master/crc32.c + *

+ */ +@SdkInternalApi +public final class CrcCombineChecksumUtil { + + public static final int CRC_SIZE = 32; + + private CrcCombineChecksumUtil() { + } + + /** + * Generates the combine matrices for CRC calculations. + * + * @param polynomial The CRC polynomial. + * @return A 2D array representing the combine matrices. + */ + public static long[][] generateCombineMatrices(long polynomial) { + long[][] combineMatrices = new long[CRC_SIZE][CRC_SIZE]; + initializeFirstMatrix(combineMatrices, polynomial); + deriveRemainingMatrices(combineMatrices); + return combineMatrices; + } + + /** + * Combines two CRC values into a single CRC using the specified combine matrices. + * + * The combination is performed using Galois Field arithmetic to effectively merge + * two CRC checksums that correspond to two separate data blocks. This method allows + * calculating the CRC for the concatenated data without having to recompute the CRC + * from scratch, which can significantly improve performance for large datasets. + *

+ * THIS COMBINE FUNCTION HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. + * The code comes from + * https://github.com/luvit/zlib/blob/master/crc32.c. + *

+ * @param crc1 The first CRC value. + * @param crc2 The second CRC value. + * @param originalLengthOfCrc2 The length of the original data for the second CRC. + * This represents the length of data used to compute {@code crc2}. + * @param combineMatrices The combine matrices used for combining CRCs. + * These matrices are precomputed to facilitate efficient combination. + * @return The combined CRC value representing the CRC for the concatenated data of both CRC values. + * @throws IllegalArgumentException if {@code originalLengthOfCrc2} is negative. + */ + public static long combine(long crc1, long crc2, long originalLengthOfCrc2, long[][] combineMatrices) { + Validate.isNotNegative(originalLengthOfCrc2, "The length of the original data for the " + + "second CRC value must be positive."); + if (originalLengthOfCrc2 == 0) { + return crc1; + } + int matrixIndex = 2; + while (originalLengthOfCrc2 != 0) { + ++matrixIndex; + if ((originalLengthOfCrc2 & 1) != 0) { + crc1 = gf2MatrixTimes(combineMatrices[matrixIndex], crc1); + } + originalLengthOfCrc2 >>= 1; + } + crc1 ^= crc2; + return crc1; + } + + /** + * Multiplies a Galois Field matrix with a vector. + * + * @param matrix The matrix to be multiplied. + * @param vector The vector to be multiplied. + * @return The result of the multiplication. + */ + private static long gf2MatrixTimes(long[] matrix, long vector) { + long sum = 0; + for (long l : matrix) { + if (vector == 0) { + break; + } + if ((vector & 1) != 0) { + sum ^= l; + } + vector >>= 1; + } + return sum; + } + + private static void initializeFirstMatrix(long[][] combineMatrices, long polynomial) { + combineMatrices[0][0] = polynomial; + long row = 1; + for (int i = 1; i < CRC_SIZE; i++) { + combineMatrices[0][i] = row; + row <<= 1; + } + } + + /** + * Derives the remaining matrices for the combination process. + * + * @param combineMatrices The combine matrices to be derived. + */ + private static void deriveRemainingMatrices(long[][] combineMatrices) { + for (int i = 0; i < CRC_SIZE - 1; i++) { + for (int j = 0; j < CRC_SIZE; j++) { + combineMatrices[i + 1][j] = gf2MatrixTimes(combineMatrices[i], combineMatrices[i][j]); + } + } + } + +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java new file mode 100644 index 000000000000..ff0f564527e6 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineFunction.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; + +/** + * Functional interface for combining two CRC values along with the data length. + */ +@SdkInternalApi +@FunctionalInterface +public interface CrcCombineFunction { + /** + * Combines two CRC values along with the data length to produce the resulting combined checksum. + * + * @param crc1 The first CRC value. + * @param crc2 The second CRC value. + * @param length The length of the data. + * @return The combined CRC value. + */ + long combine(long crc1, long crc2, long length); +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java new file mode 100644 index 000000000000..c1a27dfec4e2 --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcCombineOnMarkChecksum.java @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static software.amazon.awssdk.utils.NumericUtils.longToByte; + +import java.util.function.BiFunction; +import java.util.zip.Checksum; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.SdkChecksum; + +/** + * Class that provides functionality for combining CRC checksums with mark and reset capabilities. + * + *

+ * This class allows combining two CRC values, making it efficient to update and restore the checksum + * during mark and reset operations. It is particularly useful in scenarios where a checksum needs to + * be marked, potentially reset to a previous state, and combined with additional data. + *

+ * + *

+ * The class maintains an internal {@link Checksum} instance and uses a {@link BiFunction} to combine + * two CRC values when necessary. The combine function is applied to a pair of CRC values, along with + * the length of the data, to produce the resulting combined checksum. + *

+ */ +@SdkInternalApi +public class CrcCombineOnMarkChecksum implements SdkChecksum { + private final Checksum crc; + private long dataLengthForChecksum = 0; + private Long crcAtMark; + private long markedDataLength = 0; + private boolean isResetDone = false; + + private final CrcCombineFunction crcCombineFunction; + + + public CrcCombineOnMarkChecksum(Checksum checksum, + CrcCombineFunction crcCombineFunction) { + this.crc = checksum; + this.crcCombineFunction = crcCombineFunction; + } + + @Override + public byte[] getChecksumBytes() { + byte[] valueBytes = longToByte(getValue()); + return new byte[] { valueBytes[4], valueBytes[5], valueBytes[6], valueBytes[7] }; + } + + @Override + public void mark(int readLimit) { + if (dataLengthForChecksum > 0) { + saveMarkState(); + } + } + + @Override + public void update(int b) { + crc.update(b); + dataLengthForChecksum += 1; + } + + @Override + public void update(byte[] b, int off, int len) { + crc.update(b, off, len); + dataLengthForChecksum += len; + } + + @Override + public long getValue() { + if (canRestoreMarkedState()) { + return crcCombineFunction.combine(crcAtMark, crc.getValue(), dataLengthForChecksum - markedDataLength); + } + return crc.getValue(); + } + + @Override + public void reset() { + if (crcAtMark != null) { + crc.reset(); + dataLengthForChecksum = markedDataLength; + } else { + crc.reset(); + dataLengthForChecksum = 0; + } + isResetDone = true; + } + + private void saveMarkState() { + crcAtMark = crc.getValue(); + markedDataLength = dataLengthForChecksum; + } + + private boolean canRestoreMarkedState() { + return crcAtMark != null && isResetDone; + } + +} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java index d2a96c7377b1..3b7a49456aa9 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32CChecksum.java @@ -18,7 +18,6 @@ import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; - /* * THIS FILE HAS BEEN MODIFIED FROM THE ORIGINAL VERSION. * @@ -563,6 +562,9 @@ public final class SdkCrc32CChecksum implements Checksum, Cloneable { 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 }; + private static final long POLYNOMIAL = 0x82F63B78; + + private static final long[][] COMBINE_MATRICES = CrcCombineChecksumUtil.generateCombineMatrices(POLYNOMIAL); /** * the current CRC value, bit-flipped */ @@ -580,6 +582,19 @@ public static SdkCrc32CChecksum create() { return new SdkCrc32CChecksum(); } + /** + * Combines the CRCs of two parts.Please refer {@link CrcCombineChecksumUtil#combine(long, long, long, long[][])} + * + * + * @param crc1 The CRC of the first part. + * @param crc2 The CRC of the second part. + * @param originalLengthOfCrc2 The length of the second part's CRC before combining. + * @return The combined CRC. + */ + public static long combine(long crc1, long crc2 , long originalLengthOfCrc2) { + return CrcCombineChecksumUtil.combine(crc1, crc2, originalLengthOfCrc2, COMBINE_MATRICES); + } + @Override public long getValue() { long ret = crc; diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java index bd180c900ac4..7149695c16bf 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/SdkCrc32Checksum.java @@ -554,6 +554,11 @@ public final class SdkCrc32Checksum implements Checksum, Cloneable { 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 }; + + private static final long POLYNOMIAL = 0xEDB88320L; + + private static final long[][] COMBINE_MATRICES = CrcCombineChecksumUtil.generateCombineMatrices(POLYNOMIAL); + /** * the current CRC value, bit-flipped */ @@ -573,6 +578,18 @@ public static SdkCrc32Checksum create() { return new SdkCrc32Checksum(); } + /** + * Combines two CRC32 values into a single CRC using the specified combine matrices. + * Please refer {@link CrcCombineChecksumUtil#combine(long, long, long, long[][])} + * + * @param crcPair A pair of CRC values to combine. + * @param originalLengthOfCrc32 The length of the original data. + * @return The combined CRC value. + */ + public static long combine(Long crcPairOne, Long crcPair, long originalLengthOfCrc32) { + return CrcCombineChecksumUtil.combine(crcPairOne, crcPair, originalLengthOfCrc32, COMBINE_MATRICES); + } + @Override public long getValue() { return ~crc & 0xffffffffL; diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java new file mode 100644 index 000000000000..04386fa95f80 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/ConstructorCacheTest.java @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import java.lang.reflect.Constructor; +import java.util.Optional; +import software.amazon.awssdk.checksums.internal.ConstructorCache; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConstructorCacheTest { + + private ConstructorCache constructorCache; + + @BeforeEach + void setUp() { + constructorCache = new ConstructorCache(); + } + + @Test + void testGetConstructor_existingClassWithNoArgsConstructor() { + // Test with a known class that has a no-args constructor, e.g., String + Optional> constructor = constructorCache.getConstructor("java.lang.String"); + assertTrue(constructor.isPresent()); + assertEquals(String.class, constructor.get().getDeclaringClass()); + } + + @Test + void testGetConstructor_existingClassWithoutNoArgsConstructor() { + // Test with a class that doesn't have a no-args constructor + Optional> constructor = constructorCache.getConstructor("java.util.Scanner"); + assertFalse(constructor.isPresent()); + } + + @Test + void testGetConstructor_nonExistingClass() { + // Test with a non-existing class name + Optional> constructor = constructorCache.getConstructor("non.existent.ClassName"); + assertFalse(constructor.isPresent()); + } +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java new file mode 100644 index 000000000000..5e9a5d1e54fd --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CRTBasedCRC32CChecksumTest.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; + +public class CRTBasedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createCrtCrc32C(); + } +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java new file mode 100644 index 000000000000..9b408aa4ab75 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32CChecksumTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + + +import java.util.stream.Stream; +import org.junit.jupiter.params.provider.Arguments; + +abstract class Crc32CChecksumTest extends CrcChecksumBase { + + protected static Stream provideParametersForCrcCheckSumValues() { + return Stream.of( + Arguments.of("00000000000000000000000000000000a245d57d") + ); + } + + protected static Stream provideValidateEncodedBase64ForCrc() { + return Stream.of( + Arguments.of("Nks/tw==") + ); + } + + protected static Stream provideValidateMarkAndResetForCrc() { + return Stream.of( + Arguments.of("Nks/tw==") + ); + } + + protected static Stream provideValidateMarkForCrc() { + return Stream.of( + Arguments.of("crUfeA==") + ); + } + + protected static Stream provideValidateSingleMarksForCrc() { + return Stream.of( + Arguments.of("eNkvgQ==") + ); + } + + protected static Stream provideValidateMultipleMarksForCrc() { + return Stream.of( + Arguments.of("Bf1liQ==") + ); + } + + protected static Stream provideValidateResetWithoutMarkForCrc() { + return Stream.of( + Arguments.of("eNkvgQ==") + ); + } + +} \ No newline at end of file diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java new file mode 100644 index 000000000000..061b4f48f499 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Crc32ChecksumTest.java @@ -0,0 +1,71 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.provider.Arguments; + +class Crc32ChecksumTest extends CrcChecksumBase { + + @BeforeEach + public void setUp() { + sdkChecksum = new Crc32Checksum(); + } + + protected static Stream provideParametersForCrcCheckSumValues() { + return Stream.of( + Arguments.of("000000000000000000000000000000001fc2e6d2") + ); + } + + protected static Stream provideValidateEncodedBase64ForCrc() { + return Stream.of( + Arguments.of("NSRBwg==") + ); + } + + protected static Stream provideValidateMarkAndResetForCrc() { + return Stream.of( + Arguments.of("NSRBwg==") + ); + } + + protected static Stream provideValidateMarkForCrc() { + return Stream.of( + Arguments.of("i9aeUg==") + ); + } + + protected static Stream provideValidateSingleMarksForCrc() { + return Stream.of( + Arguments.of("0OA5ag==") + ); + } + + protected static Stream provideValidateMultipleMarksForCrc() { + return Stream.of( + Arguments.of("xSjLBA==") + ); + } + + protected static Stream provideValidateResetWithoutMarkForCrc() { + return Stream.of( + Arguments.of("0OA5ag==") + ); + } + +} \ No newline at end of file diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java new file mode 100644 index 000000000000..c9ad393bb092 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/CrcChecksumBase.java @@ -0,0 +1,112 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.utils.BinaryUtils; + +abstract class CrcChecksumBase { + + protected SdkChecksum sdkChecksum; + + static final String TEST_STRING = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + @ParameterizedTest + @MethodSource("provideParametersForCrcCheckSumValues") + void crcCheckSumValues(String expectedChecksum) throws UnsupportedEncodingException { + byte[] bytes = TEST_STRING.getBytes("UTF-8"); + sdkChecksum.update(bytes, 0, bytes.length); + assertEquals(expectedChecksum, getAsString(sdkChecksum.getChecksumBytes())); + } + + @ParameterizedTest + @MethodSource("provideValidateEncodedBase64ForCrc") + void validateEncodedBase64ForCrc(String expectedChecksum) { + sdkChecksum.update("abc".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMarkAndResetForCrc") + void validateMarkAndResetForCrc(String expectedChecksum) { + sdkChecksum.update("ab".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("xyz".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("c".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMarkForCrc") + void validateMarkForCrc(String expectedChecksum) { + sdkChecksum.update("Hello ".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(4); + sdkChecksum.update("world".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateSingleMarksForCrc") + void validateSingleMarksForCrc(String expectedChecksum) { + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateMultipleMarksForCrc") + void validateMultipleMarksForCrc(String expectedChecksum) { + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(3); + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.mark(5); + sdkChecksum.update("gamma".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("delta".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + @ParameterizedTest + @MethodSource("provideValidateResetWithoutMarkForCrc") + void validateResetWithoutMarkForCrc(String expectedChecksum) { + sdkChecksum.update("beta".getBytes(StandardCharsets.UTF_8)); + sdkChecksum.reset(); + sdkChecksum.update("alpha".getBytes(StandardCharsets.UTF_8)); + String toBase64 = BinaryUtils.toBase64(sdkChecksum.getChecksumBytes()); + assertEquals(expectedChecksum, toBase64); + } + + private String getAsString(byte[] checksumBytes) { + return String.format("%040x", new BigInteger(1, checksumBytes)); + } +} + + diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java new file mode 100644 index 000000000000..80810970e4d3 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +public class DefaultCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.create(); + } + + + @AfterEach + void tearDown() { + sdkChecksum = null; + // Try to force garbage collection to clear weak references + System.gc(); + System.runFinalization(); + } + +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java similarity index 83% rename from core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java rename to core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java index ac0a2b7791b7..82e768af16ed 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithmTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultChecksumAlgorithmTest.java @@ -13,17 +13,19 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.checksums; +package software.amazon.awssdk.checksums.internal; import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; public class DefaultChecksumAlgorithmTest { @Test public void hasCRC32C() { - assertEquals("CRC32C", DefaultChecksumAlgorithm.CRC32C.algorithmId()); + Assertions.assertEquals("CRC32C", DefaultChecksumAlgorithm.CRC32C.algorithmId()); } @Test diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java new file mode 100644 index 000000000000..1acc96dc167a --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java @@ -0,0 +1,30 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.condition.DisabledOnJre; +import org.junit.jupiter.api.condition.JRE; + +@DisabledOnJre(JRE.JAVA_8) +public class Java9BasedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createJavaCrc32C(); + } + +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java new file mode 100644 index 000000000000..98189d8bc28d --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import org.junit.jupiter.api.BeforeEach; + +public class SdkImplmenetedCRC32CChecksumTest extends Crc32CChecksumTest { + + @BeforeEach + public void setUp() { + sdkChecksum = Crc32cProvider.createSdkBasedCrc32C(); + } +} diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java index 95109d368165..f3eb940f1bf3 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java @@ -17,6 +17,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; @@ -34,7 +35,7 @@ import java.io.UncheckedIOException; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import software.amazon.awssdk.checksums.internal.Crc32CChecksum; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; import software.amazon.awssdk.checksums.internal.Md5Checksum; @@ -58,7 +59,7 @@ public void fromChecksumAlgorithm_mapsToCorrectSdkChecksum() { assertEquals(Sha256Checksum.class, fromChecksumAlgorithm(SHA256).getClass()); assertEquals(Sha1Checksum.class, fromChecksumAlgorithm(SHA1).getClass()); assertEquals(Crc32Checksum.class, fromChecksumAlgorithm(CRC32).getClass()); - assertEquals(Crc32CChecksum.class, fromChecksumAlgorithm(CRC32C).getClass()); + assertInstanceOf(SdkChecksum.class, fromChecksumAlgorithm(CRC32C)); assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); assertEquals(Crc64NvmeChecksum.class, fromChecksumAlgorithm(CRC64NVME).getClass()); diff --git a/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java b/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java index d5907cd21374..a9b7735260fa 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/ClassLoaderHelper.java @@ -120,7 +120,7 @@ public static Class loadClass(String fqcn, boolean classesFirst, * Attempt to get the current thread's class loader and fallback to the system classloader if null * @return a {@link ClassLoader} or null if none found */ - private static ClassLoader contextClassLoader() { + public static ClassLoader contextClassLoader() { ClassLoader threadClassLoader = Thread.currentThread().getContextClassLoader(); if (threadClassLoader != null) { return threadClassLoader;