From e11ee40e3cd63983671e4bc2f8f59b188ac5e53e Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 9 Feb 2023 14:26:56 +0800 Subject: [PATCH] GH-20272: [C++] Bump version of bundled AWS SDK (#33808) ### Rationale for this change Bump AWS SDK version to 1.10.55. ### What changes are included in this PR? Bump AWS SDK version to 1.10.55. * Closes: #20272 Lead-authored-by: Jin Shang Co-authored-by: Sutou Kouhei Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 238 ++++++++++++++++++-- cpp/src/arrow/filesystem/CMakeLists.txt | 44 ++-- cpp/src/arrow/filesystem/s3fs_test.cc | 26 +-- cpp/thirdparty/versions.txt | 50 +++- 4 files changed, 281 insertions(+), 77 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index dd1cf41346e9f..58fd1de3ed677 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -433,6 +433,22 @@ else() ) endif() +if(DEFINED ENV{ARROW_AWS_C_AUTH_URL}) + set(AWS_C_AUTH_SOURCE_URL "$ENV{ARROW_AWS_C_AUTH_URL}") +else() + set_urls(AWS_C_AUTH_SOURCE_URL + "https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_CAL_URL}) + set(AWS_C_CAL_SOURCE_URL "$ENV{ARROW_AWS_C_CAL_URL}") +else() + set_urls(AWS_C_CAL_SOURCE_URL + "https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWS_C_COMMON_URL}) set(AWS_C_COMMON_SOURCE_URL "$ENV{ARROW_AWS_C_COMMON_URL}") else() @@ -441,11 +457,11 @@ else() ) endif() -if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) - set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") +if(DEFINED ENV{ARROW_AWS_C_COMPRESSION_URL}) + set(AWS_C_COMPRESSION_SOURCE_URL "$ENV{ARROW_AWS_C_COMPRESSION_URL}") else() - set_urls(AWS_CHECKSUMS_SOURCE_URL - "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + set_urls(AWS_C_COMPRESSION_SOURCE_URL + "https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" ) endif() @@ -457,6 +473,62 @@ else() ) endif() +if(DEFINED ENV{ARROW_AWS_C_HTTP_URL}) + set(AWS_C_HTTP_SOURCE_URL "$ENV{ARROW_AWS_C_HTTP_URL}") +else() + set_urls(AWS_C_HTTP_SOURCE_URL + "https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_IO_URL}) + set(AWS_C_IO_SOURCE_URL "$ENV{ARROW_AWS_C_IO_URL}") +else() + set_urls(AWS_C_IO_SOURCE_URL + "https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_MQTT_URL}) + set(AWS_C_MQTT_SOURCE_URL "$ENV{ARROW_AWS_C_MQTT_URL}") +else() + set_urls(AWS_C_MQTT_SOURCE_URL + "https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_S3_URL}) + set(AWS_C_S3_SOURCE_URL "$ENV{ARROW_AWS_C_S3_URL}") +else() + set_urls(AWS_C_S3_SOURCE_URL + "https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_C_SDKUTILS_URL}) + set(AWS_C_SDKUTILS_SOURCE_URL "$ENV{ARROW_AWS_C_SDKUTILS_URL}") +else() + set_urls(AWS_C_SDKUTILS_SOURCE_URL + "https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_CHECKSUMS_URL}) + set(AWS_CHECKSUMS_SOURCE_URL "$ENV{ARROW_AWS_CHECKSUMS_URL}") +else() + set_urls(AWS_CHECKSUMS_SOURCE_URL + "https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + ) +endif() + +if(DEFINED ENV{ARROW_AWS_CRT_CPP_URL}) + set(AWS_CRT_CPP_SOURCE_URL "$ENV{ARROW_AWS_CRT_CPP_URL}") +else() + set_urls(AWS_CRT_CPP_SOURCE_URL + "https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz" + ) +endif() + if(DEFINED ENV{ARROW_AWSSDK_URL}) set(AWSSDK_SOURCE_URL "$ENV{ARROW_AWSSDK_URL}") else() @@ -651,6 +723,13 @@ else() "${THIRDPARTY_MIRROR_URL}/rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_S2N_TLS_URL}) + set(S2N_TLS_SOURCE_URL "$ENV{ARROW_S2N_TLS_URL}") +else() + set_urls(S2N_TLS_SOURCE_URL + "https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz") +endif() + if(DEFINED ENV{ARROW_SNAPPY_URL}) set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}") else() @@ -4649,11 +4728,6 @@ macro(build_awssdk) -DENABLE_UNITY_BUILD=ON "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") - if(NOT MSVC) - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582 - "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations") - endif() # provide hint for AWS SDK to link with the already located openssl get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY) @@ -4689,7 +4763,17 @@ macro(build_awssdk) aws-cpp-sdk-cognito-identity aws-cpp-sdk-s3 aws-cpp-sdk-core + aws-crt-cpp + aws-c-s3 + aws-c-auth + aws-c-mqtt + aws-c-http + aws-c-compression + aws-c-sdkutils aws-c-event-stream + aws-c-io + aws-c-cal + s2n-tls aws-checksums aws-c-common) set(AWSSDK_LIBRARIES) @@ -4701,6 +4785,11 @@ macro(build_awssdk) set(_AWSSDK_STATIC_LIBRARY "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) + if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a + set(_AWSSDK_STATIC_LIBRARY + "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endif() if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) else() @@ -4732,35 +4821,146 @@ macro(build_awssdk) DEPENDS aws_c_common_ep) add_dependencies(AWS::aws-checksums aws_checksums_ep) + set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) + if(APPLE AND NOT OPENSSL_ROOT_DIR) + find_program(BREW brew) + if(BREW) + execute_process(COMMAND ${BREW} --prefix "openssl@1.1" + OUTPUT_VARIABLE OPENSSL11_BREW_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(OPENSSL11_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL11_BREW_PREFIX}) + else() + execute_process(COMMAND ${BREW} --prefix "openssl" + OUTPUT_VARIABLE OPENSSL_BREW_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(OPENSSL_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX}) + endif() + endif() + endif() + endif() + if(OPENSSL_ROOT_DIR) + # For Findcrypto.cmake in s2n-tls. + list(APPEND S2N_TLS_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${OPENSSL_ROOT_DIR}) + endif() + externalproject_add(s2n_tls_ep + ${EP_COMMON_OPTIONS} + URL ${S2N_TLS_SOURCE_URL} + URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} + BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY}) + add_dependencies(AWS::s2n-tls s2n_tls_ep) + + externalproject_add(aws_c_cal_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_CAL_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-cal aws_c_cal_ep) + + externalproject_add(aws_c_io_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_IO_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} + DEPENDS aws_c_common_ep s2n_tls_ep aws_c_cal_ep) + add_dependencies(AWS::aws-c-io aws_c_io_ep) + externalproject_add(aws_c_event_stream_ep ${EP_COMMON_OPTIONS} URL ${AWS_C_EVENT_STREAM_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep) + DEPENDS aws_checksums_ep aws_c_io_ep) add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) - set(AWSSDK_PATCH_COMMAND) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER - "10") - # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750 - set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g" - "/cmake/compiler_settings.cmake") - endif() + externalproject_add(aws_c_sdkutils_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_SDKUTILS_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep) + + externalproject_add(aws_c_compression_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_COMPRESSION_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY} + DEPENDS aws_c_common_ep) + add_dependencies(AWS::aws-c-compression aws_c_compression_ep) + + externalproject_add(aws_c_http_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_HTTP_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} + DEPENDS aws_c_io_ep aws_c_compression_ep) + add_dependencies(AWS::aws-c-http aws_c_http_ep) + + externalproject_add(aws_c_mqtt_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_MQTT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} + DEPENDS aws_c_http_ep) + add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) + + externalproject_add(aws_c_auth_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_AUTH_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY} + DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep) + add_dependencies(AWS::aws-c-auth aws_c_auth_ep) + + externalproject_add(aws_c_s3_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_C_S3_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY} + DEPENDS aws_checksums_ep aws_c_auth_ep) + add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) + + externalproject_add(aws_crt_cpp_ep + ${EP_COMMON_OPTIONS} + URL ${AWS_CRT_CPP_SOURCE_URL} + URL_HASH "SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} + BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY} + DEPENDS aws_c_auth_ep + aws_c_cal_ep + aws_c_common_ep + aws_c_event_stream_ep + aws_c_http_ep + aws_c_io_ep + aws_c_mqtt_ep + aws_c_s3_ep + aws_checksums_ep) + add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) externalproject_add(awssdk_ep ${EP_COMMON_OPTIONS} URL ${AWSSDK_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} ${AWS_CPP_SDK_S3_STATIC_LIBRARY} ${AWS_CPP_SDK_STS_STATIC_LIBRARY} - DEPENDS aws_c_event_stream_ep) + DEPENDS aws_crt_cpp_ep) add_dependencies(toolchain awssdk_ep) foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 6888231a35a6e..04b8b1e355749 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -48,6 +48,19 @@ if(ARROW_GCS) endif() if(ARROW_S3) + set(ARROW_S3_BUILD_TESTS ${ARROW_BUILD_TESTS}) + get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) + # We disable S3 filesystem tests with bundled (static linked) AWS + # C++ SDK because it's difficult to initialize underlying S3 library + # properly. + if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY") + set(ARROW_S3_BUILD_TESTS OFF) + endif() +else() + set(ARROW_S3_BUILD_TESTS OFF) +endif() + +if(ARROW_S3_BUILD_TESTS) add_arrow_test(s3fs_test SOURCES s3fs_test.cc @@ -57,33 +70,10 @@ if(ARROW_S3) EXTRA_LINK_LIBS Boost::filesystem Boost::system) - if(TARGET arrow-s3fs-test) - set(ARROW_S3FS_TEST_COMPILE_DEFINITIONS) - get_target_property(AWS_CPP_SDK_S3_TYPE aws-cpp-sdk-s3 TYPE) - # We need to initialize AWS C++ SDK for direct use (not via - # arrow::fs::S3FileSystem) in arrow-s3fs-test if we use static AWS - # C++ SDK. Because AWS C++ SDK has internal static variables that - # aren't shared in libarrow and arrow-s3fs-test. It means that - # arrow::fs::InitializeS3() doesn't initialize AWS C++ SDK that is - # directly used in arrow-s3fs-test. - # - # But it seems that internal static variables in AWS C++ SDK are - # shared on macOS even if we link static AWS C++ SDK to both - # libarrow and arrow-s3fs-test. So we don't need to initialize AWS - # C++ SDK in arrow-s3fs-test on macOS. - if(AWS_CPP_SDK_S3_TYPE STREQUAL "STATIC_LIBRARY" AND NOT APPLE) - list(APPEND ARROW_S3FS_TEST_COMPILE_DEFINITIONS "AWS_CPP_SDK_S3_NOT_SHARED") - endif() - target_compile_definitions(arrow-s3fs-test - PRIVATE ${ARROW_S3FS_TEST_COMPILE_DEFINITIONS}) - endif() - - if(ARROW_BUILD_TESTS) - add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc) - target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS} - ${GFLAGS_LIBRARIES} GTest::gtest) - add_dependencies(arrow-tests arrow-s3fs-narrative-test) - endif() + add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc) + target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS} + ${GFLAGS_LIBRARIES} GTest::gtest) + add_dependencies(arrow-tests arrow-s3fs-narrative-test) if(ARROW_BUILD_BENCHMARKS AND ARROW_PARQUET) add_arrow_benchmark(s3fs_benchmark diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index c6957fa933ec8..e7de2ddd30e11 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -150,28 +150,8 @@ class AwsTestMixin : public ::testing::Test { // EC2 metadata endpoint AwsTestMixin() : ec2_metadata_disabled_guard_("AWS_EC2_METADATA_DISABLED", "true") {} - void SetUp() override { -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - auto aws_log_level = Aws::Utils::Logging::LogLevel::Fatal; - aws_options_.loggingOptions.logLevel = aws_log_level; - aws_options_.loggingOptions.logger_create_fn = [&aws_log_level] { - return std::make_shared(aws_log_level); - }; - Aws::InitAPI(aws_options_); -#endif - } - - void TearDown() override { -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - Aws::ShutdownAPI(aws_options_); -#endif - } - private: EnvVarGuard ec2_metadata_disabled_guard_; -#ifdef AWS_CPP_SDK_S3_NOT_SHARED - Aws::SDKOptions aws_options_; -#endif }; class S3TestMixin : public AwsTestMixin { @@ -191,7 +171,11 @@ class S3TestMixin : public AwsTestMixin { ASSERT_OK(connect_status); } - void TearDown() override { AwsTestMixin::TearDown(); } + void TearDown() override { + client_.reset(); // Aws::S3::S3Client destruction relies on AWS SDK, so it must be + // reset before Aws::ShutdownAPI + AwsTestMixin::TearDown(); + } protected: void InitServerAndClient() { diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 69073f3a47581..92509a3dd3fee 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,14 +25,32 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWSSDK_BUILD_VERSION=1.8.133 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=d6c495bc06be5e21dac716571305d77437e7cfd62a2226b8fe48d9ab5785a8d6 -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.12 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=394723034b81cc7cd528401775bc7aca2b12c7471c92350c80a0e2fb9d2909fe -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.6.9 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=928a3e36f24d1ee46f9eec360ec5cebfe8b9b8994fe39d4fa74ff51aebb12717 -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.1.5 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=f1b423a487b5d6dca118bfc0d0c6cc596dc476b282258a3228e73a8f730422d4 +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c +ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a +ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 +ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae +ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f +ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 +ARROW_AWSSDK_BUILD_VERSION=1.10.55 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de ARROW_BOOST_BUILD_VERSION=1.81.0 ARROW_BOOST_BUILD_SHA256_CHECKSUM=9e0ffae35528c35f90468997bc8d99500bf179cbae355415a89a600c38e13574 ARROW_BROTLI_BUILD_VERSION=v1.0.9 @@ -83,6 +101,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.1.9 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 ARROW_SUBSTRAIT_BUILD_VERSION=v0.20.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=5ceaa559ccef29a7825b5e5d4b5e7eed384830294f08bec913feecdd903a94cf +ARROW_S2N_TLS_BUILD_VERSION=v1.3.35 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d ARROW_THRIFT_BUILD_VERSION=0.16.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=f460b5c1ca30d8918ff95ea3eb6291b3951cf518553566088f3f2be8981f6209 ARROW_UCX_BUILD_VERSION=1.12.1 @@ -103,10 +123,19 @@ ARROW_ZSTD_BUILD_SHA256_CHECKSUM=f7de13462f7a82c29ab865820149e778cbfe01087b3a55b # given version. DEPENDENCIES=( "ARROW_ABSL_URL absl-${ARROW_ABSL_BUILD_VERSION}.tar.gz https://github.com/abseil/abseil-cpp/archive/${ARROW_ABSL_BUILD_VERSION}.tar.gz" - "ARROW_AWSSDK_URL aws-sdk-cpp-${ARROW_AWSSDK_BUILD_VERSION}.tar.gz https://github.com/aws/aws-sdk-cpp/archive/${ARROW_AWSSDK_BUILD_VERSION}.tar.gz" - "ARROW_AWS_CHECKSUMS_URL aws-checksums-${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_AUTH_URL aws-c-auth-${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-auth/archive/${ARROW_AWS_C_AUTH_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_CAL_URL aws-c-cal-${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-cal/archive/${ARROW_AWS_C_CAL_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_COMMON_URL aws-c-common-${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-common/archive/${ARROW_AWS_C_COMMON_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_COMPRESSION_URL aws-c-compression-${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-compression/archive/${ARROW_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" "ARROW_AWS_C_EVENT_STREAM_URL aws-c-event-stream-${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-event-stream/archive/${ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_HTTP_URL aws-c-http-${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-http/archive/${ARROW_AWS_C_HTTP_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_IO_URL aws-c-io-${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-io/archive/${ARROW_AWS_C_IO_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_MQTT_URL aws-c-mqtt-${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-mqtt/archive/${ARROW_AWS_C_MQTT_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_S3_URL aws-c-s3-${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-s3/archive/${ARROW_AWS_C_S3_BUILD_VERSION}.tar.gz" + "ARROW_AWS_C_SDKUTILS_URL aws-c-sdkutils-${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-c-sdkutils/archive/${ARROW_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" + "ARROW_AWS_CHECKSUMS_URL aws-checksums-${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-checksums/archive/${ARROW_AWS_CHECKSUMS_BUILD_VERSION}.tar.gz" + "ARROW_AWS_CRT_CPP_URL aws-crt-cpp-${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz https://github.com/awslabs/aws-crt-cpp/archive/${ARROW_AWS_CRT_CPP_BUILD_VERSION}.tar.gz" + "ARROW_AWSSDK_URL aws-sdk-cpp-${ARROW_AWSSDK_BUILD_VERSION}.tar.gz https://github.com/aws/aws-sdk-cpp/archive/${ARROW_AWSSDK_BUILD_VERSION}.tar.gz" "ARROW_BOOST_URL boost-${ARROW_BOOST_BUILD_VERSION}.tar.gz https://apache.jfrog.io/artifactory/arrow/thirdparty/7.0.0/boost_${ARROW_BOOST_BUILD_VERSION//./_}.tar.gz" "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz" "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" @@ -128,6 +157,7 @@ DEPENDENCIES=( "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-all-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz" "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz" "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz" + "ARROW_S2N_TLS_URL s2n-${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz" "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz" "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz" "ARROW_UCX_URL ucx-${ARROW_UCX_BUILD_VERSION}.tar.gz https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz"