From 8915ee75bf79cc4a5c26bf90483d7b6f704ea2a1 Mon Sep 17 00:00:00 2001 From: Dongie Agnir <261310+dagnir@users.noreply.github.com> Date: Fri, 19 Sep 2025 11:07:44 -0700 Subject: [PATCH 1/4] Reuse computed checksums across retries (#6413) * Reuse computed checksums across retries This commit adds the ability to reuse previously computed checksums for a request across retries. This ensures that if a request data stream is modified between attempts that the server will reject the request. As part of this change, the `http-auth-spi` package has been updated to expose a new interface: `PayloadChecksumStore`. This is a simple storage interface that allows signers to store and retrieve computed checksums. Additionally, a new `SignerProperty` is introduced, `SdkInternalHttpSignerProperty.CHECKSUM_CACHE` so that signers and their callers can access this cache. Note that both the interface and associated signer property are `@SdkProtectedApi` and not intended to be used by non-SDK consumers of `http-auth-spi`. Finally, this adds a dependency on `checksums-spi` for `http-auth-spi`. * Update core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> * Review comments --------- Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> --- .../signer/AwsChunkedV4aPayloadSigner.java | 34 ++- .../signer/DefaultAwsCrtV4aHttpSigner.java | 16 +- .../signer/AwsChunkedV4PayloadSigner.java | 34 ++- .../auth/aws/internal/signer/Checksummer.java | 14 +- .../signer/DefaultAwsV4HttpSigner.java | 15 +- .../internal/signer/FlexibleChecksummer.java | 16 +- .../signer/NoOpPayloadChecksumStore.java | 45 ++++ .../ChecksumTrailerProvider.java | 17 +- .../internal/signer/util/ChecksumUtil.java | 9 +- .../awssdk/http/auth/aws/TestUtils.java | 8 +- .../awssdk/http/auth/aws/crt/TestUtils.java | 7 +- .../DefaultAwsCrtV4aHttpSignerTest.java | 185 ++++++++++++- .../signer/DefaultAwsV4HttpSignerTest.java | 253 ++++++++++++++---- .../signer/FlexibleChecksummerTest.java | 72 ++++- .../ChecksumTrailerProviderTest.java | 87 ++++++ core/http-auth-spi/pom.xml | 5 + .../signer/DefaultPayloadChecksumStore.java | 46 ++++ .../auth/spi/signer/PayloadChecksumStore.java | 52 ++++ .../signer/SdkInternalHttpSignerProperty.java | 37 +++ .../spi/signer/PayloadChecksumStoreTest.java | 78 ++++++ .../SdkInternalExecutionAttribute.java | 8 + .../checksums/LegacyPayloadChecksumCache.java | 37 +++ .../pipeline/stages/HttpChecksumStage.java | 10 + .../http/pipeline/stages/SigningStage.java | 12 +- .../stages/HttpChecksumStageSraTest.java | 27 ++ .../pipeline/stages/SigningStageTest.java | 32 +++ services/s3/pom.xml | 6 + .../s3/checksums/ChecksumReuseTest.java | 163 +++++++++++ 28 files changed, 1237 insertions(+), 88 deletions(-) create mode 100644 core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/NoOpPayloadChecksumStore.java create mode 100644 core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChecksumTrailerProviderTest.java create mode 100644 core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/internal/signer/DefaultPayloadChecksumStore.java create mode 100644 core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/PayloadChecksumStore.java create mode 100644 core/http-auth-spi/src/main/java/software/amazon/awssdk/http/auth/spi/signer/SdkInternalHttpSignerProperty.java create mode 100644 core/http-auth-spi/src/test/java/software/amazon/awssdk/http/auth/spi/signer/PayloadChecksumStoreTest.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/checksums/LegacyPayloadChecksumCache.java create mode 100644 services/s3/src/test/java/software/amazon/awssdk/services/s3/checksums/ChecksumReuseTest.java diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java index 0124b2ea2c56..644e204f34b0 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java @@ -35,12 +35,15 @@ import software.amazon.awssdk.http.Header; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope; +import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; +import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Pair; import software.amazon.awssdk.utils.StringInputStream; import software.amazon.awssdk.utils.Validate; @@ -51,16 +54,20 @@ */ @SdkInternalApi public final class AwsChunkedV4aPayloadSigner implements V4aPayloadSigner { + private static final Logger LOG = Logger.loggerFor(AwsChunkedV4aPayloadSigner.class); private final CredentialScope credentialScope; private final int chunkSize; private final ChecksumAlgorithm checksumAlgorithm; + private final PayloadChecksumStore payloadChecksumStore; private final List>> preExistingTrailers = new ArrayList<>(); private AwsChunkedV4aPayloadSigner(Builder builder) { this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope"); this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize"); this.checksumAlgorithm = builder.checksumAlgorithm; + this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() : + builder.payloadChecksumStore; } public static Builder builder() { @@ -241,21 +248,41 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil return; } String checksumHeaderName = checksumHeaderName(checksumAlgorithm); + + String cachedChecksum = getCachedChecksum(); + + if (cachedChecksum != null) { + LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value", + checksumAlgorithm.algorithmId(), checksumHeaderName)); + builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum))); + return; + } + SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm); ChecksumInputStream checksumInputStream = new ChecksumInputStream( builder.inputStream(), Collections.singleton(sdkChecksum) ); - TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName); + TrailerProvider checksumTrailer = + new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore); builder.inputStream(checksumInputStream).addTrailer(checksumTrailer); } + private String getCachedChecksum() { + byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm); + if (checksumBytes != null) { + return BinaryUtils.toBase64(checksumBytes); + } + return null; + } + static final class Builder { private CredentialScope credentialScope; private Integer chunkSize; private ChecksumAlgorithm checksumAlgorithm; + private PayloadChecksumStore payloadChecksumStore; public Builder credentialScope(CredentialScope credentialScope) { this.credentialScope = credentialScope; @@ -272,6 +299,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { return this; } + public Builder checksumCache(PayloadChecksumStore checksumCache) { + this.payloadChecksumStore = checksumCache; + return this; + } + public AwsChunkedV4aPayloadSigner build() { return new AwsChunkedV4aPayloadSigner(this); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java index ec756d35053e..a77385b5c036 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java @@ -33,6 +33,7 @@ import static software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils.sanitizeCredentials; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; import java.time.Clock; import java.time.Duration; @@ -47,11 +48,13 @@ import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.internal.signer.Checksummer; import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope; +import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore; import software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner; import software.amazon.awssdk.http.auth.aws.signer.RegionSet; import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest; import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest; import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.http.auth.spi.signer.SignRequest; import software.amazon.awssdk.http.auth.spi.signer.SignedRequest; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; @@ -70,7 +73,7 @@ public final class DefaultAwsCrtV4aHttpSigner implements AwsV4aHttpSigner { @Override public SignedRequest sign(SignRequest request) { - Checksummer checksummer = checksummer(request, null); + Checksummer checksummer = checksummer(request, null, checksumCache(request)); V4aProperties v4aProperties = v4aProperties(request); AwsSigningConfig signingConfig = signingConfig(request, v4aProperties); V4aPayloadSigner payloadSigner = v4aPayloadSigner(request, v4aProperties); @@ -104,7 +107,7 @@ private static V4aProperties v4aProperties(BaseSignRequest request, + SignRequest request, V4aProperties v4aProperties) { boolean isPayloadSigning = isPayloadSigning(request); @@ -117,6 +120,7 @@ private static V4aPayloadSigner v4aPayloadSigner( .credentialScope(v4aProperties.getCredentialScope()) .chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES) .checksumAlgorithm(request.property(CHECKSUM_ALGORITHM)) + .checksumCache(checksumCache(request)) .build(); } @@ -252,4 +256,12 @@ private static V4aRequestSigningResult sign(SdkHttpRequest request, HttpRequest signingResult.getSignature(), signingConfig); } + + private static PayloadChecksumStore checksumCache(SignRequest request) { + PayloadChecksumStore cache = request.property(CHECKSUM_CACHE); + if (cache == null) { + return NoOpPayloadChecksumStore.create(); + } + return cache; + } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java index 391894b28b39..4f43701f9bf5 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java @@ -45,7 +45,9 @@ import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; +import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.Pair; import software.amazon.awssdk.utils.Validate; @@ -55,16 +57,20 @@ */ @SdkInternalApi public final class AwsChunkedV4PayloadSigner implements V4PayloadSigner { + private static final Logger LOG = Logger.loggerFor(AwsChunkedV4PayloadSigner.class); private final CredentialScope credentialScope; private final int chunkSize; private final ChecksumAlgorithm checksumAlgorithm; + private final PayloadChecksumStore payloadChecksumStore; private final List>> preExistingTrailers = new ArrayList<>(); private AwsChunkedV4PayloadSigner(Builder builder) { this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope"); this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize"); this.checksumAlgorithm = builder.checksumAlgorithm; + this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() : + builder.payloadChecksumStore; } public static Builder builder() { @@ -259,22 +265,43 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil if (checksumAlgorithm == null) { return; } + String checksumHeaderName = checksumHeaderName(checksumAlgorithm); + + String cachedChecksum = getCachedChecksum(); + + if (cachedChecksum != null) { + LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value", + checksumAlgorithm.algorithmId(), checksumHeaderName)); + builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum))); + return; + } + SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm); ChecksumInputStream checksumInputStream = new ChecksumInputStream( builder.inputStream(), Collections.singleton(sdkChecksum) ); - TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName); + TrailerProvider checksumTrailer = + new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore); builder.inputStream(checksumInputStream).addTrailer(checksumTrailer); } + private String getCachedChecksum() { + byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm); + if (checksumBytes != null) { + return BinaryUtils.toBase64(checksumBytes); + } + return null; + } + static class Builder { private CredentialScope credentialScope; private Integer chunkSize; private ChecksumAlgorithm checksumAlgorithm; + private PayloadChecksumStore payloadChecksumStore; public Builder credentialScope(CredentialScope credentialScope) { this.credentialScope = credentialScope; @@ -291,6 +318,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { return this; } + public Builder checksumCache(PayloadChecksumStore payloadChecksumStore) { + this.payloadChecksumStore = payloadChecksumStore; + return this; + } + public AwsChunkedV4PayloadSigner build() { return new AwsChunkedV4PayloadSigner(this); } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java index 071ed2bdec9c..3c3efb1b65ad 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java @@ -29,6 +29,7 @@ import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.BinaryUtils; /** @@ -47,8 +48,9 @@ public interface Checksummer { * Get a default implementation of a checksummer, which calculates the SHA-256 checksum and places it in the * x-amz-content-sha256 header. */ - static Checksummer create() { + static Checksummer create(PayloadChecksumStore cache) { return new FlexibleChecksummer( + cache, option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex).build() ); } @@ -57,9 +59,10 @@ static Checksummer create() { * Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the SHA-256 checksum. It places * the SHA-256 checksum in x-amz-content-sha256 header, and the given checksum-algorithm in the x-amz-checksum-[name] header. */ - static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm) { + static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm, PayloadChecksumStore cache) { if (checksumAlgorithm != null) { return new FlexibleChecksummer( + cache, option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex) .build(), option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm) @@ -82,9 +85,12 @@ static Checksummer forPrecomputed256Checksum(String precomputedSha256) { * given checksum string. It places the precomputed checksum in x-amz-content-sha256 header, and the given checksum-algorithm * in the x-amz-checksum-[name] header. */ - static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorithm checksumAlgorithm) { + static Checksummer forFlexibleChecksum(String precomputedSha256, + ChecksumAlgorithm checksumAlgorithm, + PayloadChecksumStore cache) { if (checksumAlgorithm != null) { return new FlexibleChecksummer( + cache, option().headerName(X_AMZ_CONTENT_SHA256).algorithm(new ConstantChecksumAlgorithm(precomputedSha256)) .formatter(b -> new String(b, StandardCharsets.UTF_8)).build(), option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm) @@ -96,7 +102,7 @@ static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorit } static Checksummer forNoOp() { - return new FlexibleChecksummer(); + return new FlexibleChecksummer(NoOpPayloadChecksumStore.create()); } /** diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java index 8ad79a12007e..3865fe4b714b 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java @@ -24,6 +24,7 @@ import static software.amazon.awssdk.http.auth.aws.internal.signer.util.OptionalDependencyLoaderUtil.getEventStreamV4PayloadSigner; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION; import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER; +import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE; import java.time.Clock; import java.time.Duration; @@ -38,6 +39,7 @@ import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest; import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest; import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.http.auth.spi.signer.SignRequest; import software.amazon.awssdk.http.auth.spi.signer.SignedRequest; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; @@ -55,7 +57,7 @@ public final class DefaultAwsV4HttpSigner implements AwsV4HttpSigner { @Override public SignedRequest sign(SignRequest request) { - Checksummer checksummer = checksummer(request, null); + Checksummer checksummer = checksummer(request, null, checksumCache(request)); V4Properties v4Properties = v4Properties(request); V4RequestSigner v4RequestSigner = v4RequestSigner(request, v4Properties); V4PayloadSigner payloadSigner = v4PayloadSigner(request, v4Properties); @@ -140,7 +142,7 @@ private static Checksummer asyncChecksummer(BaseSignRequest request) { + PayloadChecksumStore cache = request.property(CHECKSUM_CACHE); + if (cache == null) { + return NoOpPayloadChecksumStore.create(); + } + return cache; + } } diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java index 2cf8e2cfef26..6c1539462788 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream; import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumSubscriber; +import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore; import software.amazon.awssdk.utils.Validate; /** @@ -44,10 +45,12 @@ */ @SdkInternalApi public final class FlexibleChecksummer implements Checksummer { + private final PayloadChecksumStore cache; private final Collection