apacheGH-20272: [C++] Bump version of bundled AWS SDK (apache#33808)

### Rationale for this change Bump AWS SDK version to 1.10.55. ### What changes are included in this PR? Bump AWS SDK version to 1.10.55. * Closes: apache#20272 Lead-authored-by: Jin Shang <shangjin1997@gmail.com> Co-authored-by: Sutou Kouhei <kou@clear-code.com> Co-authored-by: Sutou Kouhei <kou@cozmixng.org> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
sjperkins · Feb 10, 2023 · 09e0392 · 09e0392
1 parent a080977
commit 09e0392
Show file tree

Hide file tree

Showing 4 changed files with 281 additions and 77 deletions.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -433,6 +433,22 @@ else()
   )
 endif()
 
+if(DEFINED ENV{ARROW_AWS_C_AUTH_URL})
+  set(AWS_C_AUTH_SOURCE_URL "$ENV{ARROW_AWS_C_AUTH_URL}")
+else()
+  set_urls(AWS_C_AUTH_SOURCE_URL
+           "https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_CAL_URL})
+  set(AWS_C_CAL_SOURCE_URL "$ENV{ARROW_AWS_C_CAL_URL}")
+else()
+  set_urls(AWS_C_CAL_SOURCE_URL
+           "https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
 if(DEFINED ENV{ARROW_AWS_C_COMMON_URL})
   set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}")
 else()
@@ -441,11 +457,11 @@ else()
   )
 endif()
 
-if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL})
-  set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}")
+if(DEFINED ENV{ARROW_AWS_C_COMPRESSION_URL})
+  set(AWS_C_COMPRESSION_SOURCE_URL "$ENV{ARROW_AWS_C_COMPRESSION_URL}")
 else()
-  set_urls(AWS_CHECKSUMS_SOURCE_URL
-           "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz"
+  set_urls(AWS_C_COMPRESSION_SOURCE_URL
+           "https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz"
   )
 endif()
 
@@ -457,6 +473,62 @@ else()
   )
 endif()
 
+if(DEFINED ENV{ARROW_AWS_C_HTTP_URL})
+  set(AWS_C_HTTP_SOURCE_URL "$ENV{ARROW_AWS_C_HTTP_URL}")
+else()
+  set_urls(AWS_C_HTTP_SOURCE_URL
+           "https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_IO_URL})
+  set(AWS_C_IO_SOURCE_URL "$ENV{ARROW_AWS_C_IO_URL}")
+else()
+  set_urls(AWS_C_IO_SOURCE_URL
+           "https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_MQTT_URL})
+  set(AWS_C_MQTT_SOURCE_URL "$ENV{ARROW_AWS_C_MQTT_URL}")
+else()
+  set_urls(AWS_C_MQTT_SOURCE_URL
+           "https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_S3_URL})
+  set(AWS_C_S3_SOURCE_URL "$ENV{ARROW_AWS_C_S3_URL}")
+else()
+  set_urls(AWS_C_S3_SOURCE_URL
+           "https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_C_SDKUTILS_URL})
+  set(AWS_C_SDKUTILS_SOURCE_URL "$ENV{ARROW_AWS_C_SDKUTILS_URL}")
+else()
+  set_urls(AWS_C_SDKUTILS_SOURCE_URL
+           "https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL})
+  set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}")
+else()
+  set_urls(AWS_CHECKSUMS_SOURCE_URL
+           "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
+if(DEFINED ENV{ARROW_AWS_CRT_CPP_URL})
+  set(AWS_CRT_CPP_SOURCE_URL "$ENV{ARROW_AWS_CRT_CPP_URL}")
+else()
+  set_urls(AWS_CRT_CPP_SOURCE_URL
+           "https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz"
+  )
+endif()
+
 if(DEFINED ENV{ARROW_AWSSDK_URL})
   set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}")
 else()
@@ -651,6 +723,13 @@ else()
            "${THIRDPARTY_MIRROR_URL}/rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz")
 endif()
 
+if(DEFINED ENV{ARROW_S2N_TLS_URL})
+  set(S2N_TLS_SOURCE_URL "$ENV{ARROW_S2N_TLS_URL}")
+else()
+  set_urls(S2N_TLS_SOURCE_URL
+           "https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz")
+endif()
+
 if(DEFINED ENV{ARROW_SNAPPY_URL})
   set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}")
 else()
@@ -4649,11 +4728,6 @@ macro(build_awssdk)
       -DENABLE_UNITY_BUILD=ON
       "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}"
       "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}")
-  if(NOT MSVC)
-    list(APPEND AWSSDK_COMMON_CMAKE_ARGS
-         # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582
-         "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations")
-  endif()
 
   # provide hint for AWS SDK to link with the already located openssl
   get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY)
@@ -4689,7 +4763,17 @@ macro(build_awssdk)
       aws-cpp-sdk-cognito-identity
       aws-cpp-sdk-s3
       aws-cpp-sdk-core
+      aws-crt-cpp
+      aws-c-s3
+      aws-c-auth
+      aws-c-mqtt
+      aws-c-http
+      aws-c-compression
+      aws-c-sdkutils
       aws-c-event-stream
+      aws-c-io
+      aws-c-cal
+      s2n-tls
       aws-checksums
       aws-c-common)
   set(AWSSDK_LIBRARIES)
@@ -4701,6 +4785,11 @@ macro(build_awssdk)
     set(_AWSSDK_STATIC_LIBRARY
         "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
     )
+    if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a
+      set(_AWSSDK_STATIC_LIBRARY
+          "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
+    endif()
     if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")
       set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB})
     else()
@@ -4732,35 +4821,146 @@ macro(build_awssdk)
                       DEPENDS aws_c_common_ep)
   add_dependencies(AWS::aws-checksums aws_checksums_ep)
 
+  set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS})
+  if(APPLE AND NOT OPENSSL_ROOT_DIR)
+    find_program(BREW brew)
+    if(BREW)
+      execute_process(COMMAND ${BREW} --prefix "openssl@1.1"
+                      OUTPUT_VARIABLE OPENSSL11_BREW_PREFIX
+                      OUTPUT_STRIP_TRAILING_WHITESPACE)
+      if(OPENSSL11_BREW_PREFIX)
+        set(OPENSSL_ROOT_DIR ${OPENSSL11_BREW_PREFIX})
+      else()
+        execute_process(COMMAND ${BREW} --prefix "openssl"
+                        OUTPUT_VARIABLE OPENSSL_BREW_PREFIX
+                        OUTPUT_STRIP_TRAILING_WHITESPACE)
+        if(OPENSSL_BREW_PREFIX)
+          set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX})
+        endif()
+      endif()
+    endif()
+  endif()
+  if(OPENSSL_ROOT_DIR)
+    # For Findcrypto.cmake in s2n-tls.
+    list(APPEND S2N_TLS_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${OPENSSL_ROOT_DIR})
+  endif()
+  externalproject_add(s2n_tls_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${S2N_TLS_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY})
+  add_dependencies(AWS::s2n-tls s2n_tls_ep)
+
+  externalproject_add(aws_c_cal_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_CAL_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY}
+                      DEPENDS aws_c_common_ep)
+  add_dependencies(AWS::aws-c-cal aws_c_cal_ep)
+
+  externalproject_add(aws_c_io_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_IO_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY}
+                      DEPENDS aws_c_common_ep s2n_tls_ep aws_c_cal_ep)
+  add_dependencies(AWS::aws-c-io aws_c_io_ep)
+
   externalproject_add(aws_c_event_stream_ep
                       ${EP_COMMON_OPTIONS}
                       URL ${AWS_C_EVENT_STREAM_SOURCE_URL}
                       URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}"
                       CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
                       BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY}
-                      DEPENDS aws_checksums_ep)
+                      DEPENDS aws_checksums_ep aws_c_io_ep)
   add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep)
 
-  set(AWSSDK_PATCH_COMMAND)
-  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER
-                                              "10")
-    # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750
-    set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g"
-                             "<SOURCE_DIR>/cmake/compiler_settings.cmake")
-  endif()
+  externalproject_add(aws_c_sdkutils_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_SDKUTILS_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY}
+                      DEPENDS aws_c_common_ep)
+  add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep)
+
+  externalproject_add(aws_c_compression_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_COMPRESSION_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY}
+                      DEPENDS aws_c_common_ep)
+  add_dependencies(AWS::aws-c-compression aws_c_compression_ep)
+
+  externalproject_add(aws_c_http_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_HTTP_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY}
+                      DEPENDS aws_c_io_ep aws_c_compression_ep)
+  add_dependencies(AWS::aws-c-http aws_c_http_ep)
+
+  externalproject_add(aws_c_mqtt_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_MQTT_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY}
+                      DEPENDS aws_c_http_ep)
+  add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep)
+
+  externalproject_add(aws_c_auth_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_AUTH_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY}
+                      DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep)
+  add_dependencies(AWS::aws-c-auth aws_c_auth_ep)
+
+  externalproject_add(aws_c_s3_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_C_S3_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY}
+                      DEPENDS aws_checksums_ep aws_c_auth_ep)
+  add_dependencies(AWS::aws-c-s3 aws_c_s3_ep)
+
+  externalproject_add(aws_crt_cpp_ep
+                      ${EP_COMMON_OPTIONS}
+                      URL ${AWS_CRT_CPP_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY}
+                      DEPENDS aws_c_auth_ep
+                              aws_c_cal_ep
+                              aws_c_common_ep
+                              aws_c_event_stream_ep
+                              aws_c_http_ep
+                              aws_c_io_ep
+                              aws_c_mqtt_ep
+                              aws_c_s3_ep
+                              aws_checksums_ep)
+  add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep)
 
   externalproject_add(awssdk_ep
                       ${EP_COMMON_OPTIONS}
                       URL ${AWSSDK_SOURCE_URL}
                       URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}"
                       CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
-                      PATCH_COMMAND ${AWSSDK_PATCH_COMMAND}
                       BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_CORE_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_S3_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_STS_STATIC_LIBRARY}
-                      DEPENDS aws_c_event_stream_ep)
+                      DEPENDS aws_crt_cpp_ep)
   add_dependencies(toolchain awssdk_ep)
   foreach(_AWSSDK_LIB ${_AWSSDK_LIBS})
     if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-")

diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt
@@ -48,6 +48,19 @@ if(ARROW_GCS)
 endif()
 
 if(ARROW_S3)
+  set(ARROW_S3_BUILD_TESTS ${ARROW_BUILD_TESTS})
+  get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE)
+  # We disable S3 filesystem tests with bundled (static linked) AWS
+  # C++ SDK because it's difficult to initialize underlying S3 library
+  # properly.
+  if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY")
+    set(ARROW_S3_BUILD_TESTS OFF)
+  endif()
+else()
+  set(ARROW_S3_BUILD_TESTS OFF)
+endif()
+
+if(ARROW_S3_BUILD_TESTS)
   add_arrow_test(s3fs_test
                  SOURCES
                  s3fs_test.cc
@@ -57,33 +70,10 @@ if(ARROW_S3)
                  EXTRA_LINK_LIBS
                  Boost::filesystem
                  Boost::system)
-  if(TARGET arrow-s3fs-test)
-    set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS)
-    get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE)
-    # We need to initialize AWS C++ SDK for direct use (not via
-    # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS
-    # C++ SDK. Because AWS C++ SDK has internal static variables that
-    # aren't shared in libarrow and arrow-s3fs-test. It means that
-    # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is
-    # directly used in arrow-s3fs-test.
-    #
-    # But it seems that internal static variables in AWS C++ SDK are
-    # shared on macOS even if we link static AWS C++ SDK to both
-    # libarrow and arrow-s3fs-test. So we don't need to initialize AWS
-    # C++ SDK in arrow-s3fs-test on macOS.
-    if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE)
-      list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_NOT_SHARED")
-    endif()
-    target_compile_definitions(arrow-s3fs-test
-                               PRIVATE ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS})
-  endif()
-
-  if(ARROW_BUILD_TESTS)
-    add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
-    target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
-                          ${GFLAGS_LIBRARIES} GTest::gtest)
-    add_dependencies(arrow-tests arrow-s3fs-narrative-test)
-  endif()
+  add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
+  target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
+                        ${GFLAGS_LIBRARIES} GTest::gtest)
+  add_dependencies(arrow-tests arrow-s3fs-narrative-test)
 
   if(ARROW_BUILD_BENCHMARKS AND ARROW_PARQUET)
     add_arrow_benchmark(s3fs_benchmark