From 4cfe9fab031a615450babf9ca89abd013f22232b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 28 Jun 2023 05:55:40 +0900 Subject: [PATCH] GH-35879: [C++] Bump bundled google-cloud-cpp to 2.12.0 (#36119) ### Rationale for this change The version will fix #35318. ### What changes are included in this PR? Use the latest released version. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * Closes: #35879 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 16 +++++++++++++++- cpp/thirdparty/versions.txt | 4 ++-- r/R/filesystem.R | 3 ++- r/configure.win | 2 +- r/src/filesystem.cpp | 13 +++++++++++++ r/tests/testthat/test-gcs.R | 10 ++++++++-- 6 files changed, 41 insertions(+), 7 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9f67f1e52f683..f7233846c18ef 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -623,6 +623,7 @@ if(DEFINED ENV{ARROW_GOOGLE_CLOUD_CPP_URL}) else() set_urls(google_cloud_cpp_storage_SOURCE_URL "https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz" + "${THIRDPARTY_MIRROR_URL}/google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz" ) endif() @@ -4188,6 +4189,7 @@ macro(build_google_cloud_cpp_storage) set_property(TARGET google-cloud-cpp::common PROPERTY INTERFACE_LINK_LIBRARIES absl::base + absl::cord absl::memory absl::optional absl::span @@ -4253,13 +4255,25 @@ macro(build_google_cloud_cpp_storage) absl::bad_variant_access absl::base absl::civil_time + absl::cord + absl::cord_internal + absl::cordz_functions + absl::cordz_info + absl::cordz_handle + absl::debugging_internal + absl::demangle_internal + absl::exponential_biased absl::int128 absl::log_severity + absl::malloc_internal absl::raw_logging_internal absl::spinlock_wait + absl::stacktrace + absl::str_format_internal absl::strings absl::strings_internal - absl::str_format_internal + absl::symbolize + absl::synchronization absl::throw_delegate absl::time absl::time_zone diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 8c8bd5d711142..a229c340414a5 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -69,8 +69,8 @@ ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf ARROW_GLOG_BUILD_VERSION=v0.5.0 ARROW_GLOG_BUILD_SHA256_CHECKSUM=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5 -ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v2.8.0 -ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=21fb441b5a670a18bb16b6826be8e0530888d0b94320847c538d46f5a54dddbc +ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v2.12.0 +ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=8cda870803925c62de8716a765e03eb9d34249977e5cdb7d0d20367e997a55e2 ARROW_GRPC_BUILD_VERSION=v1.46.3 ARROW_GRPC_BUILD_SHA256_CHECKSUM=d6cbf22cb5007af71b61c6be316a79397469c58c82a942552a62e708bce60964 ARROW_GTEST_BUILD_VERSION=1.11.0 diff --git a/r/R/filesystem.R b/r/R/filesystem.R index f028e57425ba1..eed9e95162bc3 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -181,6 +181,7 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F #' - `retry_limit_seconds`: the maximum amount of time to spend retrying if #' the filesystem encounters errors. Default is 15 seconds. #' - `default_metadata`: default metadata to write in new objects. +#' - `project_id`: the project to use for creating buckets. #' #' @section Methods: #' @@ -562,7 +563,7 @@ GcsFileSystem$create <- function(anonymous = FALSE, retry_limit_seconds = 15, .. valid_opts <- c( "access_token", "expiration", "json_credentials", "endpoint_override", - "scheme", "default_bucket_location", "default_metadata" + "scheme", "default_bucket_location", "default_metadata", "project_id" ) invalid_opts <- setdiff(names(options), valid_opts) diff --git a/r/configure.win b/r/configure.win index d9daffb5d98e0..eed0c632dea13 100755 --- a/r/configure.win +++ b/r/configure.win @@ -33,7 +33,7 @@ AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-man -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" # pkg-config --libs libcurl GCS_LIBS="-lcurl -lnormaliz -lssh2 -lgdi32 -lssl -lcrypto -lcrypt32 -lwldap32 \ - -lz -lws2_32 -lnghttp2" + -lz -lws2_32 -lnghttp2 -ldbghelp" function configure_release() { VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 4388d111b4a64..23bcb81e8faae 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -425,6 +425,15 @@ std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, gcs_opts.default_metadata = strings_to_kvm(options["default_metadata"]); } + // /// \brief The project to use for creating buckets. + // /// + // /// If not set, the library uses the GOOGLE_CLOUD_PROJECT environment + // /// variable. Most I/O operations do not need a project id, only applications + // /// that create new buckets need a project id. + if (!Rf_isNull(options["project_id"])) { + gcs_opts.project_id = cpp11::as_cpp(options["project_id"]); + } + auto io_context = MainRThread::GetInstance().CancellableIOContext(); // TODO(ARROW-16884): update when this returns Result return fs::GcsFileSystem::Make(gcs_opts, io_context); @@ -487,6 +496,10 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr out.push_back({"default_metadata"_nm = metadata}); } + if (opts.project_id.has_value()) { + out.push_back({"project_id"_nm = opts.project_id.value()}); + } + return out; } diff --git a/r/tests/testthat/test-gcs.R b/r/tests/testthat/test-gcs.R index fd173e923b277..d671c12138c60 100644 --- a/r/tests/testthat/test-gcs.R +++ b/r/tests/testthat/test-gcs.R @@ -132,7 +132,8 @@ fs <- GcsFileSystem$create( endpoint_override = sprintf("localhost:%s", testbench_port), retry_limit_seconds = 1, scheme = "http", - anonymous = TRUE # Will fail to resolve host name if anonymous isn't TRUE + anonymous = TRUE, # Will fail to resolve host name if anonymous isn't TRUE + project_id = "test-project-id" ) now <- as.character(as.numeric(Sys.time())) @@ -155,7 +156,12 @@ gcs_path <- function(...) { paste(now, ..., sep = "/") } gcs_uri <- function(...) { - template <- "gs://anonymous@%s?scheme=http&endpoint_override=localhost%s%s&retry_limit_seconds=1" + template <- paste0("gs://anonymous@%s?", + paste("scheme=http", + "endpoint_override=localhost%s%s", + "retry_limit_seconds=1", + "project_id=test-project-id", + sep = "&")) sprintf(template, gcs_path(...), "%3A", testbench_port) }