Revert "Memory management integration with common runtime libraries."

.cspell.json

@@ -13,7 +13,7 @@
         "Presigner", "xindex", "errortype", "waveout", "WAVEOUTCAPSA", "ALLOWSYNC", "WAVEHDR", "MMSYSERR",
         "WAVEFORMATEX", "Unprepare", "DDISABLE_IMDSV1", "SENDREQUEST", "threadpool",
         // AWS general
-        "Arns", "AMZN", "amzn", "Paulo", "Ningxia", "ISOB", "isob", "AWSXML", "IMDSV",
+        "Arns", "AMZN", "amzn", "Paulo", "Ningxia", "ISOB", "isob", "AWSXML", "IMDSV", "AWSSTL",
         // AWS Signature
         "SIGV", "AUTHV",
         // CMake

generated/tests/s3-gen-tests/S3EndpointProviderTests.cpp

@@ -20,8 +20,6 @@ using ExpEpProps = Aws::UnorderedMap<Aws::String, Aws::Vector<Aws::Vector<EpProp
 using ExpEpAuthScheme = Aws::Vector<EpProp>;
 using ExpEpHeaders = Aws::UnorderedMap<Aws::String, Aws::Vector<Aws::String>>;
 
-class S3EndpointProviderTests : public ::testing::TestWithParam<size_t> {};
-
 struct S3EndpointProviderEndpointTestCase
 {
     using OperationParamsFromTest = EndpointParameters;
@@ -55,7 +53,32 @@ struct S3EndpointProviderEndpointTestCase
     // Aws::Vector<OperationInput> operationInput;
 };
 
-static const Aws::Vector<S3EndpointProviderEndpointTestCase> TEST_CASES = {
+class S3EndpointProviderTests : public ::testing::TestWithParam<size_t>
+{
+public:
+    static const size_t TEST_CASES_SZ;
+protected:
+    static Aws::Vector<S3EndpointProviderEndpointTestCase> getTestCase();
+    static Aws::UniquePtrSafeDeleted<Aws::Vector<S3EndpointProviderEndpointTestCase>> TEST_CASES;
+    static void SetUpTestSuite()
+    {
+        TEST_CASES = Aws::MakeUniqueSafeDeleted<Aws::Vector<S3EndpointProviderEndpointTestCase>>(ALLOCATION_TAG, getTestCase());
+        ASSERT_TRUE(TEST_CASES) << "Failed to allocate TEST_CASES table";
+        assert(TEST_CASES->size() == TEST_CASES_SZ);
+    }
+
+    static void TearDownTestSuite()
+    {
+        TEST_CASES.reset();
+    }
+};
+
+Aws::UniquePtrSafeDeleted<Aws::Vector<S3EndpointProviderEndpointTestCase>> S3EndpointProviderTests::TEST_CASES;
+const size_t S3EndpointProviderTests::TEST_CASES_SZ = 297;
+
+Aws::Vector<S3EndpointProviderEndpointTestCase> S3EndpointProviderTests::getTestCase() {
+
+  Aws::Vector<S3EndpointProviderEndpointTestCase> test_cases = {
   /*TEST CASE 0*/
   {"region is not a valid DNS-suffix", // documentation
     {EpParam("UseFIPS", false), EpParam("Region", "a b"), EpParam("Accelerate", false), EpParam("UseDualStack", false)}, // params
@@ -2585,7 +2608,9 @@ static const Aws::Vector<S3EndpointProviderEndpointTestCase> TEST_CASES = {
     {}, // tags
     {{/*No endpoint expected*/}, /*error*/"S3Express bucket name is not a valid virtual hostable name."} // expect
   }
-};
+  };
+  return test_cases;
+}
 
 Aws::String RulesToSdkSignerName(const Aws::String& rulesSignerName)
 {
@@ -2680,9 +2705,10 @@ void ValidateOutcome(const ResolveEndpointOutcome& outcome, const S3EndpointProv
 TEST_P(S3EndpointProviderTests, EndpointProviderTest)
 {
     const size_t TEST_CASE_IDX = GetParam();
-    ASSERT_LT(TEST_CASE_IDX, TEST_CASES.size()) << "Something is wrong with the test fixture itself.";
-    const S3EndpointProviderEndpointTestCase& TEST_CASE = TEST_CASES.at(TEST_CASE_IDX);
+    ASSERT_LT(TEST_CASE_IDX, TEST_CASES->size()) << "Something is wrong with the test fixture itself.";
+    const S3EndpointProviderEndpointTestCase& TEST_CASE = TEST_CASES->at(TEST_CASE_IDX);
     SCOPED_TRACE(Aws::String("\nTEST CASE # ") + Aws::Utils::StringUtils::to_string(TEST_CASE_IDX) + ": " + TEST_CASE.documentation);
+    SCOPED_TRACE(Aws::String("\n--gtest_filter=EndpointTestsFromModel/S3EndpointProviderTests.EndpointProviderTest/") + Aws::Utils::StringUtils::to_string(TEST_CASE_IDX));
 
     std::shared_ptr<S3EndpointProvider> endpointProvider = Aws::MakeShared<S3EndpointProvider>(ALLOCATION_TAG);
     ASSERT_TRUE(endpointProvider) << "Failed to allocate/initialize S3EndpointProvider";
@@ -2724,4 +2750,4 @@ TEST_P(S3EndpointProviderTests, EndpointProviderTest)
 
 INSTANTIATE_TEST_SUITE_P(EndpointTestsFromModel,
                          S3EndpointProviderTests,
-                         ::testing::Range((size_t) 0u, TEST_CASES.size()));
+                         ::testing::Range((size_t) 0u, S3EndpointProviderTests::TEST_CASES_SZ));

src/aws-cpp-sdk-core/include/aws/core/utils/memory/stl/AWSAllocator.h

@@ -18,8 +18,65 @@
 namespace Aws
 {
 #ifdef USE_AWS_MEMORY_MANAGEMENT
+    /**
+     * Std allocator interface that is used for all STL types in CRT library
+     * in the event that Custom Memory Management is being used.
+     *
+     * Both SDK and CRT STL allocators are going to call the same SDK memory allocation methods.
+     * However, we should keep them separated
+     * to allow SDK to use the allocator when CRT is not initialized or already terminated.
+     */
+    template< typename T > using CrtAllocator = Aws::Crt::StlAllocator<T>;
+
+    /**
+     * Std allocator interface that is used for all STL types in the SDK
+     * in the event that Custom Memory Management is being used.
+     */  
+    template <typename T>
+    class Allocator : public std::allocator<T>
+    {
+    public:
+
+        typedef std::allocator<T> Base;
+
+        Allocator() throw() :
+            Base()
+        {}
+
+        Allocator(const Allocator<T>& a) throw() :
+            Base(a)
+        {}
+
+        template <class U>
+        Allocator(const Allocator<U>& a) throw() :
+            Base(a)
+        {}
 
-    template< typename T > using Allocator = Aws::Crt::StlAllocator<T>;
+        ~Allocator() throw() {}
+
+        typedef std::size_t size_type;
+
+        template<typename U>
+        struct rebind
+        {
+            typedef Allocator<U> other;
+        };
+
+        typename Base::pointer allocate(size_type n, const void *hint = nullptr)
+        {
+            AWS_UNREFERENCED_PARAM(hint);
+
+            return reinterpret_cast<typename Base::pointer>(Malloc("AWSSTL", n * sizeof(T)));
+        }
+
+        void deallocate(typename Base::pointer p, size_type n)
+        {
+            AWS_UNREFERENCED_PARAM(n);
+
+            Free(p);
+        }
+
+    };
 
 #ifdef __ANDROID__
 #if _GLIBCXX_FULLY_DYNAMIC_STRING == 0
@@ -46,6 +103,13 @@ namespace Aws
     template<typename T, typename ...ArgTypes>
     std::shared_ptr<T> MakeShared(const char* allocationTag, ArgTypes&&... args)
     {
+#ifdef USE_AWS_MEMORY_MANAGEMENT
+        Aws::Utils::Memory::MemorySystemInterface* memorySystem = Aws::Utils::Memory::GetMemorySystem();
+        // Was InitAPI forgotten or ShutdownAPI already called or Aws:: class used as static?
+        // TODO: enforce to non-conditional assert
+        AWS_ASSERT(memorySystem && "Memory system is not initialized.");
+        AWS_UNREFERENCED_PARAM(memorySystem);
+#endif
         AWS_UNREFERENCED_PARAM(allocationTag);
 
         return std::allocate_shared<T, Aws::Allocator<T>>(Aws::Allocator<T>(), std::forward<ArgTypes>(args)...);

src/aws-cpp-sdk-core/source/Aws.cpp

@@ -46,7 +46,6 @@ namespace Aws
             Aws::Utils::Memory::InitializeAWSMemorySystem(*options.memoryManagementOptions.memoryManager);
         }
 #endif // USE_AWS_MEMORY_MANAGEMENT
-        Aws::InitializeCrt();
         Aws::Client::CoreErrorsMapper::InitCoreErrorsMapper();
         if(options.loggingOptions.logLevel != Aws::Utils::Logging::LogLevel::Off)
         {
@@ -72,6 +71,7 @@ namespace Aws
             AWS_LOGSTREAM_INFO(ALLOCATION_TAG, "Initiate AWS SDK for C++ with Version:" << Aws::String(Aws::Version::GetVersionString()));
         }
 
+        Aws::InitializeCrt();
         Aws::Config::InitConfigAndCredentialsCacheManager();
 
         if (options.ioOptions.clientBootstrap_create_fn)
@@ -210,14 +210,14 @@ namespace Aws
 
         Aws::Config::CleanupConfigAndCredentialsCacheManager();
 
+        Aws::Client::CoreErrorsMapper::CleanupCoreErrorsMapper();
+        Aws::CleanupCrt();
+
         if (options.loggingOptions.logLevel != Aws::Utils::Logging::LogLevel::Off)
         {
             Aws::Utils::Logging::ShutdownCRTLogging();
             Aws::Utils::Logging::PushLogger(nullptr); // stops further logging but keeps old logger object alive
         }
-        Aws::Client::CoreErrorsMapper::CleanupCoreErrorsMapper();
-        Aws::CleanupCrt();
-
         Aws::Utils::Logging::ShutdownAWSLogging();
 #ifdef USE_AWS_MEMORY_MANAGEMENT
         if(options.memoryManagementOptions.memoryManager)

src/aws-cpp-sdk-core/source/utils/logging/DefaultLogSystem.cpp

@@ -27,6 +27,8 @@ static void LogThread(DefaultLogSystem::LogSynchronizationData* syncData, std::s
 {
     // localtime requires access to env. variables to get Timezone, which is not thread-safe
     int32_t lastRolledHour = DateTime::Now().GetHour(false /*localtime*/);
+    Aws::Vector<Aws::String> messages;
+    messages.reserve(BUFFERED_MSG_COUNT);
 
     for(;;)
     {
@@ -38,17 +40,15 @@ static void LogThread(DefaultLogSystem::LogSynchronizationData* syncData, std::s
             break;
         }
 
-        Aws::Vector<Aws::String> messages(std::move(syncData->m_queuedLogMessages));
-        syncData->m_queuedLogMessages.reserve(BUFFERED_MSG_COUNT);
-
+        std::swap(messages, syncData->m_queuedLogMessages);
         locker.unlock();
 
         if (messages.size() > 0)
         {
             if (rollLog)
             {
                 // localtime requires access to env. variables to get Timezone, which is not thread-safe
-                int32_t currentHour = DateTime::Now().GetHour(false /*localtime*/); 
+                int32_t currentHour = DateTime::Now().GetHour(false /*localtime*/);
                 if (currentHour != lastRolledHour)
                 {
                     logFile = MakeDefaultLogFile(filenamePrefix);
@@ -63,6 +63,12 @@ static void LogThread(DefaultLogSystem::LogSynchronizationData* syncData, std::s
 
             logFile->flush();
         }
+        messages.clear();
+        if(messages.capacity() > 2 * BUFFERED_MSG_COUNT)
+        {
+            messages.shrink_to_fit();
+            messages.reserve(BUFFERED_MSG_COUNT);
+        }
     }
 
     {
@@ -77,6 +83,7 @@ DefaultLogSystem::DefaultLogSystem(LogLevel logLevel, const std::shared_ptr<Aws:
     m_syncData(),
     m_loggingThread()
 {
+    m_syncData.m_queuedLogMessages.reserve(BUFFERED_MSG_COUNT);
     m_loggingThread = std::thread(LogThread, &m_syncData, logFile, "", false);
 }
 
@@ -85,6 +92,7 @@ DefaultLogSystem::DefaultLogSystem(LogLevel logLevel, const Aws::String& filenam
     m_syncData(),
     m_loggingThread()
 {
+    m_syncData.m_queuedLogMessages.reserve(BUFFERED_MSG_COUNT);
     m_loggingThread = std::thread(LogThread, &m_syncData, MakeDefaultLogFile(filenamePrefix), filenamePrefix, true);
 }
 

src/aws-cpp-sdk-core/source/utils/logging/FormattedLogSystem.cpp

@@ -141,4 +141,8 @@ void FormattedLogSystem::LogStream(LogLevel logLevel, const char* tag, const Aws
 {
     auto message = message_stream.str();
     ProcessFormattedStatement(CreateLogPrefixLine(logLevel, tag, message.size()) + std::move(message) + "\n");
+    if (LogLevel::Fatal == logLevel)
+    {
+        Flush();
+    }
 }

src/aws-cpp-sdk-core/source/utils/memory/AWSMemory.cpp

@@ -65,6 +65,11 @@ MemorySystemInterface* GetMemorySystem()
 void* Malloc(const char* allocationTag, size_t allocationSize)
 {
     Aws::Utils::Memory::MemorySystemInterface* memorySystem = Aws::Utils::Memory::GetMemorySystem();
+#ifdef USE_AWS_MEMORY_MANAGEMENT
+    // Was InitAPI forgotten or ShutdownAPI already called or Aws:: class used as static?
+    // TODO: enforce to non-conditional assert AWS_ASSERT
+    AWS_ASSERT(memorySystem && "Memory system is not initialized.");
+#endif
 
     void* rawMemory = nullptr;
     if(memorySystem != nullptr)
@@ -88,6 +93,11 @@ void Free(void* memoryPtr)
     }
 
     Aws::Utils::Memory::MemorySystemInterface* memorySystem = Aws::Utils::Memory::GetMemorySystem();
+#ifdef USE_AWS_MEMORY_MANAGEMENT
+    // Was InitAPI forgotten or ShutdownAPI already called or Aws:: class used as static?
+    // TODO: enforce to non-conditional assert
+    AWS_ASSERT(memorySystem && "Memory system is not initialized.");
+#endif
     if(memorySystem != nullptr)
     {
         memorySystem->FreeMemory(memoryPtr);
@@ -101,7 +111,7 @@ void Free(void* memoryPtr)
 static void* MemAcquire(aws_allocator* allocator, size_t size)
 {
     (void)allocator; // unused;
-    return Aws::Malloc("MemAcquire", size);
+    return Aws::Malloc("CrtMemAcquire", size);
 }
 
 static void MemRelease(aws_allocator* allocator, void* ptr)

tests/aws-cpp-sdk-s3-crt-integration-tests/BucketAndObjectOperationTest.cpp

@@ -63,20 +63,20 @@ using namespace Aws::Utils;
 
 namespace
 {
-    static Aws::String BASE_CREATE_BUCKET_TEST_NAME = "createbuckettest";
-    static Aws::String BASE_DNS_UNFRIENDLY_TEST_NAME = "dns.unfriendly";
-    static Aws::String BASE_LOCATION_BUCKET_TEST_NAME = "locbuckettest";
-    static Aws::String BASE_OBJECTS_BUCKET_NAME = "objecttest";
-    static Aws::String BASE_OBJECTS_DEFAULT_CTOR_BUCKET_NAME = "ctortest";
-    static Aws::String BASE_PUT_OBJECTS_BUCKET_NAME = "putobjecttest";
-    static Aws::String BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME = "charsetstest";
-    static Aws::String BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME = "presignedtest";
-    static Aws::String BASE_PUT_MULTIPART_BUCKET_NAME = "multiparttest";
-    static Aws::String BASE_ERRORS_TESTING_BUCKET = "errorstest";
-    static Aws::String BASE_EVENT_STREAM_TEST_BUCKET_NAME = "eventstream";
-    static Aws::String BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
-    static Aws::String BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
-    static Aws::String BASE_CHECKSUMS_BUCKET_NAME = "checksums-crt";
+    static std::string BASE_CREATE_BUCKET_TEST_NAME = "createbuckettest";
+    static std::string BASE_DNS_UNFRIENDLY_TEST_NAME = "dns.unfriendly";
+    static std::string BASE_LOCATION_BUCKET_TEST_NAME = "locbuckettest";
+    static std::string BASE_OBJECTS_BUCKET_NAME = "objecttest";
+    static std::string BASE_OBJECTS_DEFAULT_CTOR_BUCKET_NAME = "ctortest";
+    static std::string BASE_PUT_OBJECTS_BUCKET_NAME = "putobjecttest";
+    static std::string BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME = "charsetstest";
+    static std::string BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME = "presignedtest";
+    static std::string BASE_PUT_MULTIPART_BUCKET_NAME = "multiparttest";
+    static std::string BASE_ERRORS_TESTING_BUCKET = "errorstest";
+    static std::string BASE_EVENT_STREAM_TEST_BUCKET_NAME = "eventstream";
+    static std::string BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
+    static std::string BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
+    static std::string BASE_CHECKSUMS_BUCKET_NAME = "checksums-crt";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
     static const char* TEST_OBJ_KEY = "TestObjectKey";
     static const char* TEST_NOT_MODIFIED_OBJ_KEY = "TestNotModifiedObjectKey";
@@ -90,7 +90,7 @@ namespace
 
     static const int TIMEOUT_MAX = 20;
 
-    void AppendUUID(Aws::String& bucketName)
+    void AppendUUID(std::string& bucketName)
     {
         using Aws::Utils::UUID;
         Aws::StringStream s;
@@ -100,7 +100,7 @@ namespace
 
     void EnsureUniqueBucketNames()
     {
-        Aws::Vector<std::reference_wrapper<Aws::String>> TEST_BUCKETS =
+        Aws::Vector<std::reference_wrapper<std::string>> TEST_BUCKETS =
             {
               std::ref(BASE_CREATE_BUCKET_TEST_NAME),
               std::ref(BASE_DNS_UNFRIENDLY_TEST_NAME),
@@ -121,7 +121,7 @@ namespace
         for (auto& testBucketName : TEST_BUCKETS)
         {
             AppendUUID(testBucketName);
-            SCOPED_TRACE(Aws::String("EnsureUniqueBucketNames: ") + testBucketName.get());
+            SCOPED_TRACE(Aws::String("EnsureUniqueBucketNames: ") + testBucketName.get().c_str());
         }
     }
 
@@ -621,7 +621,7 @@ namespace
         //Create Client with default constructor
         Client = Aws::MakeShared<S3CrtClient>(ALLOCATION_TAG);
 
-        const Aws::String fullBucketName = CalculateBucketName(BASE_OBJECTS_DEFAULT_CTOR_BUCKET_NAME);
+        const Aws::String fullBucketName = CalculateBucketName(BASE_OBJECTS_DEFAULT_CTOR_BUCKET_NAME.c_str());
         SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
         CreateBucketRequest createBucketRequest;
         createBucketRequest.SetBucket(fullBucketName);

tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp

@@ -614,7 +614,7 @@ namespace
                 ASSERT_NE(S3Errors::VALIDATION, getObjectOutcome.GetError().GetErrorType());
                 Aws::StringStream ss;
                 ss << "https://" << expectedEndpoint << "/fakeObjectKey";
-                if (ss.str() != TestingMonitoringMetrics::s_lastUriString) {
+                if (ss.str() != TestingMonitoringMetrics::s_lastUriString.c_str()) {
                     std::cout << "Error";
                 }
                 ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());