Checksum notes rebase (#2826)

Fixes bug where specified checksums would be overwritten by SDK re-calculating checksum

---------

Co-authored-by: sbiscigl <sbiscigl@amazon.com>

src/aws-cpp-sdk-core/include/aws/core/utils/crypto/PrecalculatedHash.h

@@ -0,0 +1,37 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#pragma once
+
+#include <aws/core/Core_EXPORTS.h>
+#include <aws/core/utils/crypto/Hash.h>
+#include <aws/core/utils/crypto/HashResult.h>
+#include <aws/core/utils/Outcome.h>
+
+namespace Aws {
+    namespace Utils {
+        namespace Crypto {
+            /**
+             * A hash implementation that stores a pre-calculated hash
+             * behind the Hash interface. It is a read through wrapper
+             * around the checksum value and no hashing are actually done.
+             */
+            class AWS_CORE_API PrecalculatedHash : public Hash {
+            public:
+                explicit PrecalculatedHash(const Aws::String &hash);
+                ~PrecalculatedHash() override;
+                HashResult Calculate(const Aws::String &str) override;
+                HashResult Calculate(Aws::IStream &stream) override;
+                void Update(unsigned char *string, size_t bufferSize) override;
+                HashResult GetHash() override;
+
+            private:
+                Aws::String m_hashString;
+                HashResult m_decodedHashString;
+
+            };
+        }
+    }
+}

src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp

@@ -237,17 +237,18 @@ bool AWSAuthV4Signer::SignRequest(Aws::Http::HttpRequest& request, const char* r
             return false;
         }
 
-        if (request.GetRequestHash().second != nullptr)
+        Aws::String checksumHeaderKey = Aws::String("x-amz-checksum-") + request.GetRequestHash().first;
+        const auto headers = request.GetHeaders();
+        if (request.GetRequestHash().second != nullptr && !request.HasHeader(checksumHeaderKey.c_str()))
         {
-            Aws::String checksumHeaderKey = Aws::String("x-amz-checksum-") + request.GetRequestHash().first;
             Aws::String checksumHeaderValue;
             if (request.GetRequestHash().first == "sha256") {
                 // we already calculated the payload hash so just reverse the hex string to
                 // a ByteBuffer and Base64Encode it - otherwise we're re-hashing the content
                 checksumHeaderValue = HashingUtils::Base64Encode(HashingUtils::HexDecode(payloadHash));
             } else {
                 // if it is one of the other hashes, we must be careful if there is no content body
-                auto body = request.GetContentBody();
+                const auto& body = request.GetContentBody();
                 checksumHeaderValue = (body)
                     ? HashingUtils::Base64Encode(request.GetRequestHash().second->Calculate(*body).GetResult())
                     : HashingUtils::Base64Encode(request.GetRequestHash().second->Calculate({}).GetResult());
@@ -263,8 +264,9 @@ bool AWSAuthV4Signer::SignRequest(Aws::Http::HttpRequest& request, const char* r
         if (request.GetRequestHash().second != nullptr)
         {
             payloadHash = STREAMING_UNSIGNED_PAYLOAD_TRAILER;
-            Aws::String trailerHeaderValue = Aws::String("x-amz-checksum-") + request.GetRequestHash().first;
-            request.SetHeaderValue(Http::AWS_TRAILER_HEADER, trailerHeaderValue);
+            Aws::String checksumHeaderValue = Aws::String("x-amz-checksum-") + request.GetRequestHash().first;
+            request.DeleteHeader(checksumHeaderValue.c_str());
+            request.SetHeaderValue(Http::AWS_TRAILER_HEADER, checksumHeaderValue);
             request.SetTransferEncoding(CHUNKED_VALUE);
             request.SetHeaderValue(Http::CONTENT_ENCODING_HEADER, Http::AWS_CHUNKED_VALUE);
             if (request.HasHeader(Http::CONTENT_LENGTH_HEADER)) {

src/aws-cpp-sdk-core/source/client/AWSClient.cpp

@@ -32,6 +32,7 @@
 #include <aws/core/utils/crypto/CRC32.h>
 #include <aws/core/utils/crypto/Sha256.h>
 #include <aws/core/utils/crypto/Sha1.h>
+#include <aws/core/utils/crypto/PrecalculatedHash.h>
 #include <aws/core/utils/HashingUtils.h>
 #include <aws/core/utils/crypto/Factories.h>
 #include <aws/core/utils/event/EventStream.h>
@@ -791,56 +792,70 @@ void AWSClient::AddChecksumToRequest(const std::shared_ptr<Aws::Http::HttpReques
                               request.GetServiceSpecificParameters()->parameterMap.find("overrideChecksumDisable") !=
                               request.GetServiceSpecificParameters()->parameterMap.end();
 
-    // Request checksums
+    //Check if user has provided the checksum algorithm
     if (!checksumAlgorithmName.empty() && !shouldSkipChecksum)
     {
+        // Check if user has provided a checksum value for the specified algorithm
+        const Aws::String checksumType = "x-amz-checksum-" + checksumAlgorithmName;
+        const HeaderValueCollection &headers = request.GetHeaders();
+        const auto checksumHeader = headers.find(checksumType);
+        bool checksumValueAndAlgorithmProvided = checksumHeader != headers.end();
+
         // For non-streaming payload, the resolved checksum location is always header.
         // For streaming payload, the resolved checksum location depends on whether it is an unsigned payload, we let AwsAuthSigner decide it.
-        if (checksumAlgorithmName == "crc32")
+        if (request.IsStreaming() && checksumValueAndAlgorithmProvided)
+        {
+            const auto hash = Aws::MakeShared<Crypto::PrecalculatedHash>(AWS_CLIENT_LOG_TAG, checksumHeader->second);
+            httpRequest->SetRequestHash(checksumAlgorithmName,hash);
+        }
+        else if (checksumValueAndAlgorithmProvided){
+            httpRequest->SetHeaderValue(checksumType, checksumHeader->second);
+        }
+        else if (checksumAlgorithmName == "crc32")
         {
             if (request.IsStreaming())
             {
-                httpRequest->SetRequestHash("crc32", Aws::MakeShared<Crypto::CRC32>(AWS_CLIENT_LOG_TAG));
+                httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared<Crypto::CRC32>(AWS_CLIENT_LOG_TAG));
             }
             else
             {
-                httpRequest->SetHeaderValue("x-amz-checksum-crc32", HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request)))));
+                httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request)))));
             }
         }
         else if (checksumAlgorithmName == "crc32c")
         {
             if (request.IsStreaming())
             {
-                httpRequest->SetRequestHash("crc32c", Aws::MakeShared<Crypto::CRC32C>(AWS_CLIENT_LOG_TAG));
+                httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared<Crypto::CRC32C>(AWS_CLIENT_LOG_TAG));
             }
             else
             {
-                httpRequest->SetHeaderValue("x-amz-checksum-crc32c", HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request)))));
+                httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request)))));
             }
         }
         else if (checksumAlgorithmName == "sha256")
         {
             if (request.IsStreaming())
             {
-                httpRequest->SetRequestHash("sha256", Aws::MakeShared<Crypto::Sha256>(AWS_CLIENT_LOG_TAG));
+                httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared<Crypto::Sha256>(AWS_CLIENT_LOG_TAG));
             }
             else
             {
-                httpRequest->SetHeaderValue("x-amz-checksum-sha256", HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request)))));
+                httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request)))));
             }
         }
         else if (checksumAlgorithmName == "sha1")
         {
             if (request.IsStreaming())
             {
-                httpRequest->SetRequestHash("sha1", Aws::MakeShared<Crypto::Sha1>(AWS_CLIENT_LOG_TAG));
+                httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared<Crypto::Sha1>(AWS_CLIENT_LOG_TAG));
             }
             else
             {
-                httpRequest->SetHeaderValue("x-amz-checksum-sha1", HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request)))));
+                httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request)))));
             }
         }
-        else if (checksumAlgorithmName == "md5")
+        else if (checksumAlgorithmName == "md5" && headers.find(CONTENT_MD5_HEADER) == headers.end())
         {
             httpRequest->SetHeaderValue(Http::CONTENT_MD5_HEADER, HashingUtils::Base64Encode(HashingUtils::CalculateMD5(*(GetBodyStream(request)))));
         }

src/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp

@@ -331,8 +331,11 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata, boo
                 chunkedTrailer << "0\r\n";
                 if (request->GetRequestHash().second != nullptr)
                 {
-                    chunkedTrailer << "x-amz-checksum-" << request->GetRequestHash().first << ":"
-                        << HashingUtils::Base64Encode(request->GetRequestHash().second->GetHash().GetResult()) << "\r\n";
+                    chunkedTrailer << "x-amz-checksum-"
+                        << request->GetRequestHash().first
+                        << ":"
+                        << HashingUtils::Base64Encode(request->GetRequestHash().second->GetHash().GetResult())
+                        << "\r\n";
                 }
                 chunkedTrailer << "\r\n";
                 amountRead = chunkedTrailer.str().size();

src/aws-cpp-sdk-core/source/utils/crypto/PrecalculatedHash.cpp

@@ -0,0 +1,37 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+
+#include <aws/core/utils/crypto/PrecalculatedHash.h>
+#include <aws/core/utils/HashingUtils.h>
+
+using namespace Aws::Utils::Crypto;
+
+PrecalculatedHash::PrecalculatedHash(const Aws::String &hash) : m_hashString(hash), m_decodedHashString(HashingUtils::Base64Decode(hash)) {}
+
+PrecalculatedHash::~PrecalculatedHash() = default;
+
+HashResult PrecalculatedHash::Calculate(const Aws::String& str)
+{
+    AWS_UNREFERENCED_PARAM(str);
+    return m_decodedHashString;
+}
+
+HashResult PrecalculatedHash::Calculate(Aws::IStream& stream)
+{
+    AWS_UNREFERENCED_PARAM(stream);
+    return m_decodedHashString;
+}
+
+void PrecalculatedHash::Update(unsigned char* string, size_t bufferSize)
+{
+    AWS_UNREFERENCED_PARAM(string);
+    AWS_UNREFERENCED_PARAM(bufferSize);
+}
+
+HashResult PrecalculatedHash::GetHash()
+{
+    return m_decodedHashString;
+}

tests/aws-cpp-sdk-s3-crt-integration-tests/BucketAndObjectOperationTest.cpp

@@ -76,6 +76,7 @@ namespace
     static Aws::String BASE_EVENT_STREAM_TEST_BUCKET_NAME = "eventstream";
     static Aws::String BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
     static Aws::String BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
+    static Aws::String BASE_CHECKSUMS_BUCKET_NAME = "checksums-crt";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
     static const char* TEST_OBJ_KEY = "TestObjectKey";
     static const char* TEST_NOT_MODIFIED_OBJ_KEY = "TestNotModifiedObjectKey";
@@ -113,7 +114,8 @@ namespace
               std::ref(BASE_ERRORS_TESTING_BUCKET),
               std::ref(BASE_EVENT_STREAM_TEST_BUCKET_NAME),
               std::ref(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME),
-              std::ref(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME)
+              std::ref(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME),
+              std::ref(BASE_CHECKSUMS_BUCKET_NAME)
             };
 
         for (auto& testBucketName : TEST_BUCKETS)
@@ -159,6 +161,7 @@ namespace
             DeleteBucket(CalculateBucketName(BASE_EVENT_STREAM_TEST_BUCKET_NAME.c_str()));
             DeleteBucket(CalculateBucketName(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME.c_str()));
             DeleteBucket(CalculateBucketName(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME.c_str()));
+            DeleteBucket(CalculateBucketName(BASE_CHECKSUMS_BUCKET_NAME.c_str()));
 
             Client = nullptr;
             oregonClient = nullptr;
@@ -1400,4 +1403,114 @@ namespace
         ASSERT_FALSE(result.IsSuccess());
         ASSERT_EQ((Aws::Client::CoreErrors) result.GetError().GetErrorType(), Aws::Client::CoreErrors::NOT_INITIALIZED);
     }
+
+    TEST_F(BucketAndObjectOperationTest, PutObjectChecksum) {
+        struct ChecksumTestCase {
+            std::function<PutObjectRequest(PutObjectRequest)> chucksumRequestMutator;
+            HttpResponseCode responseCode;
+            String body;
+        };
+
+        const String fullBucketName = CalculateBucketName(BASE_CHECKSUMS_BUCKET_NAME.c_str());
+        SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
+        CreateBucketRequest createBucketRequest;
+        createBucketRequest.SetBucket(fullBucketName);
+        createBucketRequest.SetACL(BucketCannedACL::private_);
+        CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
+        AWS_ASSERT_SUCCESS(createBucketOutcome);
+
+        Vector<ChecksumTestCase> testCases{
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32).WithChecksumCRC32("Just runnin' scared each place we go");
+                },
+                HttpResponseCode::BAD_REQUEST,
+                "Just runnin' scared each place we go"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA1).WithChecksumSHA1("So afraid that he might show");
+                },
+                HttpResponseCode::BAD_REQUEST,
+                "So afraid that he might show"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA256).WithChecksumSHA256("Yeah, runnin' scared, what would I do");
+                },
+                HttpResponseCode::BAD_REQUEST,
+                "Yeah, runnin' scared, what would I do"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32C).WithChecksumCRC32C("If he came back and wanted you?");
+                },
+                HttpResponseCode::BAD_REQUEST,
+                "If he came back and wanted you?"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithContentMD5("Just runnin' scared, feelin' low");
+                },
+                HttpResponseCode::BAD_REQUEST,
+                "Just runnin' scared, feelin' low",
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
+                        .WithChecksumCRC32(HashingUtils::Base64Encode(HashingUtils::CalculateCRC32("Runnin' scared, you love him so")));
+                },
+                HttpResponseCode::OK,
+                "Runnin' scared, you love him so"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA1)
+                        .WithChecksumSHA1(HashingUtils::Base64Encode(HashingUtils::CalculateSHA1("Just runnin' scared, afraid to lose")));
+                },
+                HttpResponseCode::OK,
+                "Just runnin' scared, afraid to lose"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA256)
+                        .WithChecksumSHA256(HashingUtils::Base64Encode(HashingUtils::CalculateSHA256("If he came back, which one would you choose?")));
+                },
+                HttpResponseCode::OK,
+                "If he came back, which one would you choose?"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32C)
+                        .WithChecksumCRC32C(HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C("Then all at once he was standing there")));
+                },
+                HttpResponseCode::OK,
+                "Then all at once he was standing there"
+            },
+            {
+                [](PutObjectRequest request) -> PutObjectRequest {
+                    return request.WithContentMD5(HashingUtils::Base64Encode(HashingUtils::CalculateMD5("So sure of himself, his head in the air")));
+                },
+                HttpResponseCode::OK,
+                "So sure of himself, his head in the air"
+            }
+        };
+
+        for (const auto&testCase: testCases) {
+            auto request = testCase.chucksumRequestMutator(PutObjectRequest()
+                .WithBucket(fullBucketName)
+                .WithKey("RunningScared"));
+            std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(ALLOCATION_TAG,
+                testCase.body,
+                std::ios_base::in | std::ios_base::binary);
+            request.SetBody(body);
+            const auto response = Client->PutObject(request);
+            if (!response.IsSuccess()) {
+                ASSERT_EQ(testCase.responseCode, response.GetError().GetResponseCode());
+            } else {
+                ASSERT_EQ(testCase.responseCode, HttpResponseCode::OK);
+                ASSERT_TRUE(response.IsSuccess());
+            }
+        }
+    }
 }