diff --git a/CMakeLists.txt b/CMakeLists.txt index fee1807892..12c87dadd2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -project(pegasus) cmake_minimum_required(VERSION 3.11.0) +project(pegasus) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") # require at least gcc 7.0.0 diff --git a/run.sh b/run.sh index ab5e9852df..ad0206f8eb 100755 --- a/run.sh +++ b/run.sh @@ -211,12 +211,7 @@ function run_build() CMAKE_OPTIONS="-DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER} - -DUSE_JEMALLOC=${USE_JEMALLOC} - -DENABLE_GCOV=${ENABLE_GCOV} - -DENABLE_GPERF=${ENABLE_GPERF} - -DBoost_NO_BOOST_CMAKE=ON - -DBOOST_ROOT=${THIRDPARTY_ROOT}/output - -DBoost_NO_SYSTEM_PATHS=ON" + -DUSE_JEMALLOC=${USE_JEMALLOC}" echo "BUILD_TYPE=$BUILD_TYPE" if [ "$BUILD_TYPE" == "debug" ] @@ -258,6 +253,13 @@ function run_build() cd .. fi + CMAKE_OPTIONS="${CMAKE_OPTIONS} + -DENABLE_GCOV=${ENABLE_GCOV} + -DENABLE_GPERF=${ENABLE_GPERF} + -DBoost_NO_BOOST_CMAKE=ON + -DBOOST_ROOT=${THIRDPARTY_ROOT}/output + -DBoost_NO_SYSTEM_PATHS=ON" + echo "INFO: start build Pegasus..." BUILD_DIR="${BUILD_ROOT_DIR}/${BUILD_TYPE}_${SANITIZER}" BUILD_DIR=${BUILD_DIR%_*} diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 8645ee1a1c..967fd48857 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -17,8 +17,8 @@ # under the License. ############################################################################## -project(pegasus_thirdparties) cmake_minimum_required(VERSION 3.11.0) +project(pegasus_thirdparties) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") # require at least gcc 7.0.0 @@ -44,6 +44,12 @@ set(TP_OUTPUT ${PROJECT_SOURCE_DIR}/output) # LOG_DIR = set_property(DIRECTORY PROPERTY EP_BASE ${TP_DIR}/build) +include(ProcessorCount) +ProcessorCount(PARALLEL) +if(PARALLEL EQUAL 0) + set(PARALLEL 1) +endif() + set(OSS_URL_PREFIX "http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com") message(STATUS "Setting up third-parties...") @@ -59,7 +65,9 @@ ExternalProject_Add(boost BUILD_COMMAND ./b2 toolset=gcc cxxflags=-fPIC cxxstd=11 install INSTALL_COMMAND cp -R include/boost ${TP_OUTPUT}/include && cp -R lib ${TP_OUTPUT}/ BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) # header-only file(MAKE_DIRECTORY ${TP_OUTPUT}/include/concurrentqueue) @@ -71,7 +79,9 @@ ExternalProject_Add(concurrentqueue BUILD_COMMAND "" INSTALL_COMMAND cp -R blockingconcurrentqueue.h concurrentqueue.h lightweightsemaphore.h internal/ ${TP_OUTPUT}/include/concurrentqueue BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) # header-only file(MAKE_DIRECTORY ${TP_OUTPUT}/include/readerwriterqueue) @@ -83,7 +93,9 @@ ExternalProject_Add(readerwriterqueue BUILD_COMMAND "" INSTALL_COMMAND cp -R atomicops.h readerwriterqueue.h ${TP_OUTPUT}/include/readerwriterqueue BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(googletest URL ${OSS_URL_PREFIX}/googletest-release-1.8.0.tar.gz @@ -92,7 +104,9 @@ ExternalProject_Add(googletest CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Get_property(googletest SOURCE_DIR) set(googletest_SRC ${SOURCE_DIR}) @@ -102,7 +116,9 @@ ExternalProject_Add(gperftools URL_MD5 4e218a40a354748c50d054c285caaae8 CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} --enable-static=no --enable-frame-pointers=yes BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) set(HDFS_CLIENT_DIR "hadoop-hdfs-project/hadoop-hdfs-native-client") ExternalProject_Add(hadoop @@ -115,7 +131,9 @@ ExternalProject_Add(hadoop CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) # header-only ExternalProject_Add(rapidjson @@ -126,7 +144,9 @@ ExternalProject_Add(rapidjson BUILD_COMMAND "" INSTALL_COMMAND cp -R include/rapidjson ${TP_OUTPUT}/include BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(thrift URL ${OSS_URL_PREFIX}/thrift-0.9.3.tar.gz @@ -153,7 +173,9 @@ ExternalProject_Add(thrift -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} DEPENDS boost - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) check_cxx_compiler_flag(-Wformat-overflow COMPILER_SUPPORTS_FORMAT_OVERFLOW) if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) @@ -172,13 +194,15 @@ ExternalProject_Add(zookeeper URL_MD5 44c2a33e01931aed94ef7f3d39d0963e PATCH_COMMAND "" COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -DWITH_OPENSSL=OFF -DWITH_CYRUS_SASL=${ZOOKEEPER_WITH_CYRUS_SASL} -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - COMMAND cd zookeeper-client/zookeeper-client-c && make + COMMAND cd zookeeper-client/zookeeper-client-c && make -j${PARALLEL} COMMAND cp -R zookeeper-client/zookeeper-client-c/include/. ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/generated/zookeeper.jute.h ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/libzookeeper.a ${TP_OUTPUT}/lib && cp zookeeper-client/zookeeper-client-c/libhashtable.a ${TP_OUTPUT}/lib CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz @@ -192,8 +216,9 @@ ExternalProject_Add(libevent -DEVENT__DISABLE_SAMPLES=On -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(poco URL ${OSS_URL_PREFIX}/poco-1.11.1-release.tar.gz @@ -217,7 +242,9 @@ ExternalProject_Add(poco -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} DEPENDS boost - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(fds URL ${OSS_URL_PREFIX}/fds-1.0.0.tar.gz @@ -237,7 +264,9 @@ ExternalProject_Add(fds UPDATE_COMMAND "" # do not update DEPENDS googletest poco boost BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(fmt URL https://github.com/fmtlib/fmt/archive/refs/tags/10.1.1.tar.gz @@ -248,16 +277,9 @@ ExternalProject_Add(fmt -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - ) - -ExternalProject_Add(gflags - URL ${OSS_URL_PREFIX}/gflags-2.2.1.zip - https://github.com/gflags/gflags/archive/v2.2.1.zip - URL_MD5 2d988ef0b50939fb50ada965dafce96b - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) # civetweb is one of the dependencies of promemetheus-cpp, do not build & install ExternalProject_Add(civetweb @@ -267,7 +289,9 @@ ExternalProject_Add(civetweb CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Get_property(civetweb SOURCE_DIR) set(civetweb_SRC ${SOURCE_DIR}) @@ -310,7 +334,9 @@ ExternalProject_Add(curl CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} ${CURL_OPTIONS} BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(prometheus-cpp URL ${OSS_URL_PREFIX}/prometheus-cpp-0.7.0.tar.gz @@ -323,7 +349,9 @@ ExternalProject_Add(prometheus-cpp -DENABLE_TESTING=OFF -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) # header-only ExternalProject_Add(nlohmann_json @@ -334,7 +362,9 @@ ExternalProject_Add(nlohmann_json BUILD_COMMAND "" INSTALL_COMMAND cp -R include/nlohmann ${TP_OUTPUT}/include BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(s2geometry URL ${OSS_URL_PREFIX}/s2geometry-e8d16637a467d9f096a92a6d81a50a9b747ca828.zip @@ -350,7 +380,9 @@ ExternalProject_Add(s2geometry -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} DEPENDS googletest - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) option(USE_JEMALLOC "use jemalloc" OFF) message(STATUS "USE_JEMALLOC = ${USE_JEMALLOC}") @@ -361,13 +393,14 @@ ExternalProject_Add(jemalloc URL_MD5 3d41fbf006e6ebffd489bdb304d009ae PATCH_COMMAND patch -p0 < ${TP_DIR}/fix_jemalloc_for_m1_on_macos.patch CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} --enable-cxx --enable-stats --enable-prof - BUILD_COMMAND make + BUILD_COMMAND make -j${PARALLEL} INSTALL_COMMAND make install BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) option(ROCKSDB_PORTABLE "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU" 0) - set(ROCKSDB_OPTIONS -DFAIL_ON_WARNINGS=OFF -DWITH_BENCHMARK_TOOLS=OFF @@ -396,12 +429,15 @@ ExternalProject_Add(rocksdb URL ${OSS_URL_PREFIX}/rocksdb-v8.5.3.tar.gz https://github.com/facebook/rocksdb/archive/refs/tags/v8.5.3.tar.gz URL_MD5 f03eac50ec958a21a7cb56183afb8fe4 - PATCH_COMMAND "" + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs DEPENDS googletest jemalloc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - ${ROCKSDB_OPTIONS}) + ${ROCKSDB_OPTIONS} + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(http-parser URL ${OSS_URL_PREFIX}/http-parser-2.9.4.zip @@ -411,7 +447,9 @@ ExternalProject_Add(http-parser BUILD_COMMAND "" INSTALL_COMMAND cp -R http_parser.h ${TP_OUTPUT}/include/nodejs/http_parser.h BUILD_IN_SOURCE 1 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) ExternalProject_Add(abseil URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip @@ -420,4 +458,6 @@ ExternalProject_Add(abseil CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DABSL_FIND_GOOGLETEST=OFF -DCMAKE_CXX_STANDARD=14 - ) + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) diff --git a/thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch b/thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch new file mode 100644 index 0000000000..6fede3e3d0 --- /dev/null +++ b/thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch @@ -0,0 +1,75 @@ +From 67233eaf66ee067867265bff87e9d89fb7c46086 Mon Sep 17 00:00:00 2001 +From: Kefu Chai +Date: Mon, 18 Sep 2023 12:11:15 -0700 +Subject: [PATCH] cmake: check PORTABLE for well-known boolean representations + (#11724) + +Summary: +before 459969e9, we were using CMake `option()` to represent `PORTABLE`. so the CMake boolean representations like ON, OFF, 0 and 1 are supported. this is also the downstream package maintainers were using before v8.3.2. + +in 459969e9, this option is expanded to specify the argument of `-march` passed to compiler in order to be more flexible and hence allows user to specify CPU type directly. but in the typical use cases, user would just want to use "ON" for the best performance on the building host, and "OFF" for a portable build when it comes to a distro package maintainer. + +so, in this change, let's check for the boolean representations supported by CMake. + +Fixes https://github.com/facebook/rocksdb/issues/11558 +Signed-off-by: Kefu Chai + +Pull Request resolved: https://github.com/facebook/rocksdb/pull/11724 + +Reviewed By: anand1976 + +Differential Revision: D48827447 + +Pulled By: ajkr + +fbshipit-source-id: b2fef7076b2e90ad13a1fbec80e197841fa06d38 +--- + CMakeLists.txt | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 4e30f6631..98cbf33ea 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -254,7 +254,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") + endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") + + set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU") +-if(PORTABLE STREQUAL 1) ++if(PORTABLE MATCHES "1|ON|YES|TRUE|Y") + # Usually nothing to do; compiler default is typically the most general + if(NOT MSVC) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x") +@@ -264,14 +264,7 @@ if(PORTABLE STREQUAL 1) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64") + endif() + endif() +-elseif(PORTABLE MATCHES [^0]+) +- # Name of a CPU arch spec or feature set to require +- if(MSVC) +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}") +- else() +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}") +- endif() +-else() ++elseif(PORTABLE MATCHES "0|OFF|NO|FALSE|N") + if(MSVC) + # NOTE: No auto-detection of current CPU, but instead assume some useful + # level of optimization is supported +@@ -285,6 +278,13 @@ else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + endif() + endif() ++else() ++ # Name of a CPU arch spec or feature set to require ++ if(MSVC) ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}") ++ else() ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}") ++ endif() + endif() + + include(CheckCXXSourceCompiles) +-- +2.42.1 +