Merge branch 'oap-project:update' into update

gaoyangxiaozhu · Jul 1, 2024 · 2e41e80 · 2e41e80
2 parents dc3f229 + 3b138e5
commit 2e41e80
Show file tree

Hide file tree

Showing 90 changed files with 3,144 additions and 596 deletions.
diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml
@@ -62,6 +62,7 @@ jobs:
       Protobuf_SOURCE: BUNDLED # can be removed after #10134 is merged
       simdjson_SOURCE: BUNDLED
       xsimd_SOURCE: BUNDLED
+      Arrow_SOURCE: AUTO
       CUDA_VERSION: "12.4"
     steps:
       - uses: actions/checkout@v4

diff --git a/CMake/FindArrow.cmake b/CMake/FindArrow.cmake
@@ -0,0 +1,37 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_library(ARROW_LIB libarrow.a)
+find_library(PARQUET_LIB libparquet.a)
+find_library(ARROW_TESTING_LIB libarrow_testing.a)
+if("${ARROW_LIB}" STREQUAL "ARROW_LIB-NOTFOUND"
+   # OR "${PARQUET_LIB}" STREQUAL "PARQUET_LIB-NOTFOUND"
+   OR "${ARROW_TESTING_LIB}" STREQUAL "ARROW_TESTING_LIB-NOTFOUND")
+  set(Arrow_FOUND false)
+  return()
+endif()
+set(Arrow_FOUND true)
+
+add_library(arrow STATIC IMPORTED GLOBAL)
+add_library(parquet STATIC IMPORTED GLOBAL)
+add_library(arrow_testing STATIC IMPORTED GLOBAL)
+
+find_path(ARROW_INCLUDE_PATH arrow/api.h)
+set_target_properties(
+  arrow arrow_testing parquet PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                         ${ARROW_INCLUDE_PATH})
+set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_LIB})
+set_target_properties(parquet PROPERTIES IMPORTED_LOCATION ${PARQUET_LIB})
+set_target_properties(arrow_testing PROPERTIES IMPORTED_LOCATION
+                                               ${ARROW_TESTING_LIB})
diff --git a/third_party/cmake_modules/FindThrift.cmake → CMake/FindThrift.cmake b/third_party/cmake_modules/FindThrift.cmake → CMake/FindThrift.cmake
@@ -12,27 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# - Find Thrift (a cross platform RPC lib/tool)
+# * Find Thrift (a cross platform RPC lib/tool)
 #
 # Variables used by this module, they can change the default behaviour and need
 # to be set before calling find_package:
 #
-#  Thrift_ROOT - When set, this path is inspected instead of standard library
-#                locations as the root of the Thrift installation.
-#                The environment variable THRIFT_HOME overrides this variable.
+# Thrift_ROOT - When set, this path is inspected instead of standard library
+# locations as the root of the Thrift installation. The environment variable
+# THRIFT_HOME overrides this variable.
 #
-# This module defines
-#  Thrift_FOUND, whether Thrift is found or not
-#  Thrift_COMPILER_FOUND, whether Thrift compiler is found or not
+# This module defines Thrift_FOUND, whether Thrift is found or not
+# Thrift_COMPILER_FOUND, whether Thrift compiler is found or not
 #
-#  thrift::thrift, a library target to use Thrift
-#  thrift::compiler, a executable target to use Thrift compiler
+# thrift::thrift, a library target to use Thrift thrift::compiler, a executable
+# target to use Thrift compiler
 
 function(EXTRACT_THRIFT_VERSION)
   if(THRIFT_INCLUDE_DIR)
     file(READ "${THRIFT_INCLUDE_DIR}/thrift/config.h" THRIFT_CONFIG_H_CONTENT)
-    string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\"" THRIFT_VERSION_DEFINITION
-                 "${THRIFT_CONFIG_H_CONTENT}")
+    string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\""
+                 THRIFT_VERSION_DEFINITION "${THRIFT_CONFIG_H_CONTENT}")
     string(REGEX MATCH "[0-9.]+" Thrift_VERSION "${THRIFT_VERSION_DEFINITION}")
     set(Thrift_VERSION
         "${Thrift_VERSION}"
@@ -66,14 +65,16 @@ set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}")
 if(ARROW_THRIFT_USE_SHARED)
   set(THRIFT_LIB_NAMES thrift)
   if(CMAKE_IMPORT_LIBRARY_SUFFIX)
-    list(APPEND
-         THRIFT_LIB_NAMES
-         "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
+    list(
+      APPEND
+      THRIFT_LIB_NAMES
+      "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
     )
   endif()
-  list(APPEND
-       THRIFT_LIB_NAMES
-       "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}"
+  list(
+    APPEND
+    THRIFT_LIB_NAMES
+    "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}"
   )
 else()
   set(THRIFT_LIB_NAMES
@@ -82,40 +83,47 @@ else()
 endif()
 
 if(Thrift_ROOT)
-  find_library(THRIFT_LIB
-               NAMES ${THRIFT_LIB_NAMES}
-               PATHS ${Thrift_ROOT}
-               PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib")
-  find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h
-            PATHS ${Thrift_ROOT}
-            PATH_SUFFIXES "include")
-  find_program(THRIFT_COMPILER thrift
-               PATHS ${Thrift_ROOT}
-               PATH_SUFFIXES "bin")
+  find_library(
+    THRIFT_LIB
+    NAMES ${THRIFT_LIB_NAMES}
+    PATHS ${Thrift_ROOT}
+    PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib")
+  find_path(
+    THRIFT_INCLUDE_DIR thrift/Thrift.h
+    PATHS ${Thrift_ROOT}
+    PATH_SUFFIXES "include")
+  find_program(
+    THRIFT_COMPILER thrift
+    PATHS ${Thrift_ROOT}
+    PATH_SUFFIXES "bin")
   extract_thrift_version()
 else()
-  # THRIFT-4760: The pkgconfig files are currently only installed when using autotools.
-  # Starting with 0.13, they are also installed for the CMake-based installations of Thrift.
+  # THRIFT-4760: The pkgconfig files are currently only installed when using
+  # autotools. Starting with 0.13, they are also installed for the CMake-based
+  # installations of Thrift.
   find_package(PkgConfig QUIET)
   pkg_check_modules(THRIFT_PC thrift)
   if(THRIFT_PC_FOUND)
     set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}")
 
     list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}")
 
-    find_library(THRIFT_LIB
-                 NAMES ${THRIFT_LIB_NAMES}
-                 PATHS ${THRIFT_PC_LIBRARY_DIRS}
-                 NO_DEFAULT_PATH)
-    find_program(THRIFT_COMPILER thrift
-                 HINTS ${THRIFT_PC_PREFIX}
-                 NO_DEFAULT_PATH
-                 PATH_SUFFIXES "bin")
+    find_library(
+      THRIFT_LIB
+      NAMES ${THRIFT_LIB_NAMES}
+      PATHS ${THRIFT_PC_LIBRARY_DIRS}
+      NO_DEFAULT_PATH)
+    find_program(
+      THRIFT_COMPILER thrift
+      HINTS ${THRIFT_PC_PREFIX}
+      NO_DEFAULT_PATH
+      PATH_SUFFIXES "bin")
     set(Thrift_VERSION ${THRIFT_PC_VERSION})
   else()
-    find_library(THRIFT_LIB
-                 NAMES ${THRIFT_LIB_NAMES}
-                 PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib")
+    find_library(
+      THRIFT_LIB
+      NAMES ${THRIFT_LIB_NAMES}
+      PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib")
     find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include")
     find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin")
     extract_thrift_version()
@@ -140,14 +148,15 @@ if(Thrift_FOUND)
   else()
     add_library(thrift::thrift STATIC IMPORTED)
   endif()
-  set_target_properties(thrift::thrift
-                        PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}"
-                                   INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}")
+  set_target_properties(
+    thrift::thrift
+    PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" INTERFACE_INCLUDE_DIRECTORIES
+                                                 "${THRIFT_INCLUDE_DIR}")
   if(WIN32 AND NOT MSVC_TOOLCHAIN)
-    # We don't need this for Visual C++ because Thrift uses
-    # "#pragma comment(lib, "Ws2_32.lib")" in
-    # thrift/windows/config.h for Visual C++.
-    set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES "ws2_32")
+    # We don't need this for Visual C++ because Thrift uses "#pragma
+    # comment(lib, "Ws2_32.lib")" in thrift/windows/config.h for Visual C++.
+    set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES
+                                                    "ws2_32")
   endif()
 
   if(Thrift_COMPILER_FOUND)

diff --git a/CMake/resolve_dependency_modules/arrow.cmake b/CMake/resolve_dependency_modules/arrow.cmake
@@ -0,0 +1,16 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include_guard(GLOBAL)
+
+add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/arrow)
diff --git a/third_party/CMakeLists.txt → ...e_dependency_modules/arrow/CMakeLists.txt b/third_party/CMakeLists.txt → ...e_dependency_modules/arrow/CMakeLists.txt
@@ -11,11 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-                      "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/")
-
-include(ExternalProject)
+project(Arrow)
 
 if(VELOX_ENABLE_ARROW)
   find_package(Thrift)
@@ -24,6 +20,7 @@ if(VELOX_ENABLE_ARROW)
   else()
     set(THRIFT_SOURCE "BUNDLED")
   endif()
+
   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
   set(ARROW_CMAKE_ARGS
       -DARROW_PARQUET=OFF
@@ -74,6 +71,7 @@ if(VELOX_ENABLE_ARROW)
     CMAKE_ARGS ${ARROW_CMAKE_ARGS}
     BUILD_BYPRODUCTS ${ARROW_LIBDIR}/libarrow.a ${ARROW_LIBDIR}/libparquet.a
                      ${ARROW_LIBDIR}/libarrow_testing.a ${THRIFT_LIB})
+
   add_library(arrow STATIC IMPORTED GLOBAL)
   add_library(arrow_testing STATIC IMPORTED GLOBAL)
   add_library(parquet STATIC IMPORTED GLOBAL)

diff --git a/CMake/resolve_dependency_modules/google_cloud_cpp_storage.cmake b/CMake/resolve_dependency_modules/google_cloud_cpp_storage.cmake
@@ -13,11 +13,8 @@
 # limitations under the License.
 include_guard(GLOBAL)
 
-set_source(absl)
-resolve_dependency(absl 20240116 EXACT)
-
 set_source(gRPC)
-resolve_dependency(gRPC 1.48.1 EXACT)
+resolve_dependency(gRPC CONFIG 1.48.1 REQUIRED)
 
 set(VELOX_GOOGLE_CLOUD_CPP_BUILD_VERSION 2.22.0)
 set(VELOX_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM

diff --git a/CMake/resolve_dependency_modules/grpc.cmake b/CMake/resolve_dependency_modules/grpc.cmake
@@ -13,6 +13,9 @@
 # limitations under the License.
 include_guard(GLOBAL)
 
+set_source(absl)
+resolve_dependency(absl CONFIG REQUIRED)
+
 set(VELOX_GRPC_BUILD_VERSION 1.48.1)
 set(VELOX_GRPC_BUILD_SHA256_CHECKSUM
     320366665d19027cda87b2368c03939006a37e0388bfd1091c8d2a96fbc93bd8)

diff --git a/CMake/resolve_dependency_modules/protobuf.cmake b/CMake/resolve_dependency_modules/protobuf.cmake
@@ -16,12 +16,20 @@ include_guard(GLOBAL)
 set(VELOX_PROTOBUF_BUILD_VERSION 21.8)
 set(VELOX_PROTOBUF_BUILD_SHA256_CHECKSUM
     83ad4faf95ff9cbece7cb9c56eb3ca9e42c3497b77001840ab616982c6269fb6)
-string(
-  CONCAT
-    VELOX_PROTOBUF_SOURCE_URL
-    "https://github.com/protocolbuffers/protobuf/releases/download/"
-    "v${VELOX_PROTOBUF_BUILD_VERSION}/protobuf-all-${VELOX_PROTOBUF_BUILD_VERSION}.tar.gz"
-)
+if(${VELOX_PROTOBUF_BUILD_VERSION} LESS 22.0)
+  string(
+    CONCAT
+      VELOX_PROTOBUF_SOURCE_URL
+      "https://github.com/protocolbuffers/protobuf/releases/download/"
+      "v${VELOX_PROTOBUF_BUILD_VERSION}/protobuf-all-${VELOX_PROTOBUF_BUILD_VERSION}.tar.gz"
+  )
+else()
+  set_source(absl)
+  resolve_dependency(absl CONFIG REQUIRED)
+  string(CONCAT VELOX_PROTOBUF_SOURCE_URL
+                "https://github.com/protocolbuffers/protobuf/archive/"
+                "v${VELOX_PROTOBUF_BUILD_VERSION}.tar.gz")
+endif()
 
 resolve_dependency_url(PROTOBUF)
 
@@ -34,6 +42,8 @@ FetchContent_Declare(
   OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM)
 
 set(protobuf_BUILD_TESTS OFF)
+set(protobuf_ABSL_PROVIDER
+    "package"
+    CACHE STRING "Provider of absl library")
 FetchContent_MakeAvailable(protobuf)
 set(Protobuf_INCLUDE_DIRS ${protobuf_SOURCE_DIR}/src)
-set(Protobuf_PROTOC_EXECUTABLE "${protobuf_BINARY_DIR}/protoc")
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -442,7 +442,7 @@ if(${VELOX_BUILD_MINIMAL_WITH_DWIO}
 
   # Locate or build protobuf.
   set_source(Protobuf)
-  resolve_dependency(Protobuf 3.21.7...<4)
+  resolve_dependency(Protobuf CONFIG 3.21.7 REQUIRED)
   include_directories(${Protobuf_INCLUDE_DIRS})
 endif()
 
@@ -470,7 +470,7 @@ endif()
 
 if(VELOX_ENABLE_GCS)
   set_source(google_cloud_cpp_storage)
-  resolve_dependency(google_cloud_cpp_storage 2.22.0 EXACT)
+  resolve_dependency(google_cloud_cpp_storage CONFIG 2.22.0 REQUIRED)
   add_definitions(-DVELOX_ENABLE_GCS)
 endif()
 
@@ -568,5 +568,9 @@ if("${TREAT_WARNINGS_AS_ERRORS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
 endif()
 
-add_subdirectory(third_party)
+if(VELOX_ENABLE_ARROW)
+  set_source(Arrow)
+  resolve_dependency(Arrow)
+endif()
+
 add_subdirectory(velox)
diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh
@@ -73,6 +73,12 @@ function install_gcs-sdk-cpp {
     -DABSL_PROPAGATE_CXX_STD=ON \
     -DABSL_ENABLE_INSTALL=ON
 
+  # protobuf
+  github_checkout protocolbuffers/protobuf v21.8 --depth 1
+  cmake_install \
+    -Dprotobuf_BUILD_TESTS=OFF \
+    -Dprotobuf_ABSL_PROVIDER=package
+
   # grpc
   github_checkout grpc/grpc v1.48.1 --depth 1
   cmake_install \

diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh
@@ -187,6 +187,29 @@ function install_duckdb {
   fi
 }
 
+ARROW_VERSION=15.0.0
+
+function install_arrow {
+  wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+  cd arrow/cpp
+  cmake_install \
+    -DARROW_PARQUET=OFF \
+    -DARROW_WITH_THRIFT=ON \
+    -DARROW_WITH_LZ4=ON \
+    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_WITH_ZLIB=ON \
+    -DARROW_WITH_ZSTD=ON \
+    -DARROW_JEMALLOC=OFF \
+    -DARROW_SIMD_LEVEL=NONE \
+    -DARROW_RUNTIME_SIMD_LEVEL=NONE \
+    -DARROW_WITH_UTF8PROC=OFF \
+    -DARROW_TESTING=ON \
+    -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DARROW_BUILD_STATIC=ON \
+    -DThrift_SOURCE=BUNDLED
+}
+
 function install_cuda {
   # See https://developer.nvidia.com/cuda-downloads
   dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
@@ -209,6 +232,7 @@ function install_velox_deps {
   run_and_time install_mvfst
   run_and_time install_fbthrift
   run_and_time install_duckdb
+  run_and_time install_arrow
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.