add configuration to preserve path seperator in URIs

aws · Mar 27, 2024 · 6535a7a · 6535a7a
1 parent 4e0d5c4
commit 6535a7a
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 3 deletions.
diff --git a/src/aws-cpp-sdk-core/include/aws/core/Aws.h b/src/aws-cpp-sdk-core/include/aws/core/Aws.h
@@ -105,6 +105,14 @@ namespace Aws
          * Disable legacy URL encoding that leaves `$&,:@=` unescaped for legacy purposes.
          */
         bool compliantRfc3986Encoding;
+        /**
+         * When constructing Path segments in a URI preserve path separators instead of collapsing
+         * slashes. This is useful for aligning with other SDKs and tools on key path for S3 objects
+         * as currently the C++ SDK sanitizes the path.
+         *
+         * TODO: In the next major release, this will become the default to align better with other SDKs.
+         */
+        bool preservePathSeparators = false;
     };
 
     /**

diff --git a/src/aws-cpp-sdk-core/include/aws/core/http/URI.h b/src/aws-cpp-sdk-core/include/aws/core/http/URI.h
@@ -24,6 +24,9 @@ namespace Aws
         extern bool s_compliantRfc3986Encoding;
         AWS_CORE_API void SetCompliantRfc3986Encoding(bool compliant);
 
+        extern AWS_CORE_API bool s_preservePathSeparators;
+        AWS_CORE_API void SetPreservePathSeparators(bool preservePathSeparators);
+
         //per https://tools.ietf.org/html/rfc3986#section-3.4 there is nothing preventing servers from allowing
         //multiple values for the same key. So use a multimap instead of a map.
         typedef Aws::MultiMap<Aws::String, Aws::String> QueryStringParameterCollection;
@@ -135,7 +138,16 @@ namespace Aws
                 Aws::StringStream ss;
                 ss << pathSegments;
                 Aws::String segments = ss.str();
-                for (const auto& segment : Aws::Utils::StringUtils::Split(segments, '/'))
+                const auto splitOption = s_preservePathSeparators
+                                           ? Utils::StringUtils::SplitOptions::INCLUDE_EMPTY_SEGMENTS
+                                           : Utils::StringUtils::SplitOptions::NOT_SET;
+                // Preserve legacy behavior -- we need to remove a leading "/" if use `INCLUDE_EMPTY_SEGMENTS` is specified
+                // because string split will no longer ignore leading delimiters -- which is correct.
+                auto split = Aws::Utils::StringUtils::Split(segments, '/', splitOption);
+                if (s_preservePathSeparators && m_pathSegments.empty() && !split.empty() && split.front().empty() && !m_pathHasTrailingSlash) {
+                  split.erase(split.begin());
+                }
+                for (const auto& segment: split)
                 {
                     m_pathSegments.push_back(segment);
                 }

diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h b/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h
@@ -79,9 +79,13 @@ namespace Aws
                  */
                 NOT_SET,
                 /**
-                 * Includes empty entries in the vector returned by Split()
+                 * Deprecated use INCLUDE_EMPTY_SEGMENTS instead.
                  */
-                INCLUDE_EMPTY_ENTRIES
+                INCLUDE_EMPTY_ENTRIES,
+                /**
+                 * Include delimiters as empty segments in the split string
+                 */
+                INCLUDE_EMPTY_SEGMENTS,
             };
 
             /**
@@ -116,6 +120,13 @@ namespace Aws
              */
             static Aws::Vector<Aws::String> Split(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts, SplitOptions option);
 
+            /**
+             * Splits a string on delimeter, keeping the delimiter in the string as a empty space.
+             * @param toSplit, the original string to split
+             * @param splitOn, the delimiter you want to use.
+             */
+            static Aws::Vector<Aws::String> SplitWithSpaces(const Aws::String& toSplit, char splitOn);
+
             /**
             * Splits a string on new line characters.
             */

diff --git a/src/aws-cpp-sdk-core/source/Aws.cpp b/src/aws-cpp-sdk-core/source/Aws.cpp
@@ -155,6 +155,7 @@ namespace Aws
         Aws::Http::SetInitCleanupCurlFlag(options.httpOptions.initAndCleanupCurl);
         Aws::Http::SetInstallSigPipeHandlerFlag(options.httpOptions.installSigPipeHandler);
         Aws::Http::SetCompliantRfc3986Encoding(options.httpOptions.compliantRfc3986Encoding);
+        Aws::Http::SetPreservePathSeparators(options.httpOptions.preservePathSeparators);
         Aws::Http::InitHttp();
         Aws::InitializeEnumOverflowContainer();
         cJSON_AS4CPP_Hooks hooks;

diff --git a/src/aws-cpp-sdk-core/source/http/URI.cpp b/src/aws-cpp-sdk-core/source/http/URI.cpp
@@ -27,6 +27,9 @@ const char* SEPARATOR = "://";
 bool s_compliantRfc3986Encoding = false;
 void SetCompliantRfc3986Encoding(bool compliant) { s_compliantRfc3986Encoding = compliant; }
 
+bool s_preservePathSeparators = false;
+void SetPreservePathSeparators(bool preservePathSeparators) { s_preservePathSeparators = preservePathSeparators; }
+
 Aws::String urlEncodeSegment(const Aws::String& segment, bool rfcEncoded = false)
 {
     // consolidates legacy escaping logic into one local method

diff --git a/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp b/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp
@@ -90,6 +90,11 @@ Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char spl
 
 Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts, SplitOptions option)
 {
+    if (option == SplitOptions::INCLUDE_EMPTY_SEGMENTS)
+    {
+        return StringUtils::SplitWithSpaces(toSplit, splitOn);
+    }
+
     Aws::Vector<Aws::String> returnValues;
     Aws::StringStream input(toSplit);
     Aws::String item;
@@ -128,6 +133,21 @@ Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char spl
     return returnValues;
 }
 
+Aws::Vector<Aws::String> StringUtils::SplitWithSpaces(const Aws::String& toSplit, char splitOn)
+{
+    size_t pos = 0;
+    String split{toSplit};
+    Vector<String> returnValues;
+    while ((pos = split.find(splitOn)) != std::string::npos) {
+        returnValues.emplace_back(split.substr(0, pos));
+        split.erase(0, pos + 1);
+    }
+    if (!split.empty()) {
+        returnValues.emplace_back(split);
+    }
+    return returnValues;
+}
+
 Aws::Vector<Aws::String> StringUtils::SplitOnLine(const Aws::String& toSplit)
 {
     Aws::StringStream input(toSplit);

diff --git a/tests/aws-cpp-sdk-s3-unit-tests/S3UnitTests.cpp b/tests/aws-cpp-sdk-s3-unit-tests/S3UnitTests.cpp
@@ -124,3 +124,60 @@ TEST_F(S3UnitTest, S3UriMiddleDots) {
   const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
   EXPECT_EQ("https://bluerev.s3.us-east-1.amazonaws.com/belinda/../says", seenRequest.GetUri().GetURIString());
 }
+
+TEST_F(S3UnitTest, S3UriPathPreservationOff) {
+  auto putObjectRequest = PutObjectRequest()
+      .WithBucket("velvetunderground")
+      .WithKey("////stephanie////says////////////that////////she//wants///////to/know.txt");
+
+  std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(ALLOCATION_TAG,
+    "What country shall I say is calling From across the world?",
+    std::ios_base::in | std::ios_base::binary);
+
+  putObjectRequest.SetBody(body);
+
+  //We have to mock requset because it is used to create the return body, it actually isnt used.
+  auto mockRequest = Aws::MakeShared<Standard::StandardHttpRequest>(ALLOCATION_TAG, "mockuri", HttpMethod::HTTP_GET);
+  mockRequest->SetResponseStreamFactory([]() -> IOStream* {
+    return Aws::New<StringStream>(ALLOCATION_TAG, "response-string", std::ios_base::in | std::ios_base::binary);
+  });
+  auto mockResponse = Aws::MakeShared<Standard::StandardHttpResponse>(ALLOCATION_TAG, mockRequest);
+  mockResponse->SetResponseCode(HttpResponseCode::OK);
+  _mockHttpClient->AddResponseToReturn(mockResponse);
+
+  const auto response = _s3Client->PutObject(putObjectRequest);
+  AWS_EXPECT_SUCCESS(response);
+
+  const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
+  EXPECT_EQ("https://velvetunderground.s3.us-east-1.amazonaws.com/stephanie/says/that/she/wants/to/know.txt", seenRequest.GetUri().GetURIString());
+}
+
+TEST_F(S3UnitTest, S3UriPathPreservationOn) {
+  //Turn on path preservation
+  Aws::Http::SetPreservePathSeparators(true);
+
+  auto putObjectRequest = PutObjectRequest()
+      .WithBucket("velvetunderground")
+      .WithKey("////stephanie////says////////////that////////she//wants///////to/know.txt");
+
+  std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(ALLOCATION_TAG,
+    "What country shall I say is calling From across the world?",
+    std::ios_base::in | std::ios_base::binary);
+
+  putObjectRequest.SetBody(body);
+
+  //We have to mock requset because it is used to create the return body, it actually isnt used.
+  auto mockRequest = Aws::MakeShared<Standard::StandardHttpRequest>(ALLOCATION_TAG, "mockuri", HttpMethod::HTTP_GET);
+  mockRequest->SetResponseStreamFactory([]() -> IOStream* {
+    return Aws::New<StringStream>(ALLOCATION_TAG, "response-string", std::ios_base::in | std::ios_base::binary);
+  });
+  auto mockResponse = Aws::MakeShared<Standard::StandardHttpResponse>(ALLOCATION_TAG, mockRequest);
+  mockResponse->SetResponseCode(HttpResponseCode::OK);
+  _mockHttpClient->AddResponseToReturn(mockResponse);
+
+  const auto response = _s3Client->PutObject(putObjectRequest);
+  AWS_EXPECT_SUCCESS(response);
+
+  const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
+  EXPECT_EQ("https://velvetunderground.s3.us-east-1.amazonaws.com/////stephanie////says////////////that////////she//wants///////to/know.txt", seenRequest.GetUri().GetURIString());
+}