From b3d21412b557df956be9726bcc8e5a87d44a80b1 Mon Sep 17 00:00:00 2001 From: james94 Date: Mon, 20 May 2024 20:58:51 -0700 Subject: [PATCH 01/26] Verified Can Install & Build MiNiFi w OpenSSL, CURL, ZLIB & run minifi binary via Conan v2 --- cmake/BundledLibcURL.cmake | 146 ++++++++++++---------- cmake/BundledOpenSSL.cmake | 245 +++++++++++++++++++------------------ cmake/BundledZLIB.cmake | 97 ++++++++------- conanfile.py | 74 +++++++++++ 4 files changed, 332 insertions(+), 230 deletions(-) create mode 100644 conanfile.py diff --git a/cmake/BundledLibcURL.cmake b/cmake/BundledLibcURL.cmake index bba5f70ab0..8f50c80ea3 100644 --- a/cmake/BundledLibcURL.cmake +++ b/cmake/BundledLibcURL.cmake @@ -16,83 +16,93 @@ # under the License. function(use_bundled_curl SOURCE_DIR BINARY_DIR) - # Define patch step - set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/curl/module-path.patch") - set(PC ${Bash_EXECUTABLE} -c "set -x && \ - (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\")") - # Define byproducts - string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) - if (WIN32) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "lib/libcurl.lib") - else() - set(BYPRODUCT "lib/libcurl-d.lib") - endif() - else() - include(GNUInstallDirs) - string(REPLACE "/" ";" LIBDIR_LIST ${CMAKE_INSTALL_LIBDIR}) - list(GET LIBDIR_LIST 0 LIBDIR) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "${LIBDIR}/libcurl.a") + if(USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt libcurl external lib") + include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) + + elseif(USE_STANDALONE_CMAKE) + message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") + + + # Define patch step + set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/curl/module-path.patch") + set(PC ${Bash_EXECUTABLE} -c "set -x && \ + (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\")") + # Define byproducts + string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) + if (WIN32) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "lib/libcurl.lib") + else() + set(BYPRODUCT "lib/libcurl-d.lib") + endif() else() - set(BYPRODUCT "${LIBDIR}/libcurl-d.a") + include(GNUInstallDirs) + string(REPLACE "/" ";" LIBDIR_LIST ${CMAKE_INSTALL_LIBDIR}) + list(GET LIBDIR_LIST 0 LIBDIR) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "${LIBDIR}/libcurl.a") + else() + set(BYPRODUCT "${LIBDIR}/libcurl-d.a") + endif() endif() - endif() - # Set build options - set(CURL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/curl-install" - -DBUILD_CURL_EXE=OFF - -DBUILD_TESTING=OFF - -DBUILD_SHARED_LIBS=OFF - -DHTTP_ONLY=ON - -DCURL_CA_PATH=none - -DCURL_USE_LIBSSH2=OFF - -DUSE_LIBIDN2=OFF - -DCURL_USE_LIBPSL=OFF - -DCURL_USE_OPENSSL=ON - ) + # Set build options + set(CURL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/curl-install" + -DBUILD_CURL_EXE=OFF + -DBUILD_TESTING=OFF + -DBUILD_SHARED_LIBS=OFF + -DHTTP_ONLY=ON + -DCURL_CA_PATH=none + -DCURL_USE_LIBSSH2=OFF + -DUSE_LIBIDN2=OFF + -DCURL_USE_LIBPSL=OFF + -DCURL_USE_OPENSSL=ON + ) + + append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") - append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") + # Build project + ExternalProject_Add( + curl-external + URL "https://github.com/curl/curl/releases/download/curl-8_4_0/curl-8.4.0.tar.gz" + URL_HASH "SHA256=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" + SOURCE_DIR "${BINARY_DIR}/thirdparty/curl-src" + LIST_SEPARATOR % # This is needed for passing semicolon-separated lists + CMAKE_ARGS ${CURL_CMAKE_ARGS} + PATCH_COMMAND ${PC} + BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" + EXCLUDE_FROM_ALL TRUE + ) - # Build project - ExternalProject_Add( - curl-external - URL "https://github.com/curl/curl/releases/download/curl-8_4_0/curl-8.4.0.tar.gz" - URL_HASH "SHA256=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" - SOURCE_DIR "${BINARY_DIR}/thirdparty/curl-src" - LIST_SEPARATOR % # This is needed for passing semicolon-separated lists - CMAKE_ARGS ${CURL_CMAKE_ARGS} - PATCH_COMMAND ${PC} - BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" - EXCLUDE_FROM_ALL TRUE - ) + # Set dependencies + add_dependencies(curl-external ZLIB::ZLIB OpenSSL::SSL OpenSSL::Crypto) - # Set dependencies - add_dependencies(curl-external ZLIB::ZLIB OpenSSL::SSL OpenSSL::Crypto) + # Set variables + set(CURL_FOUND "YES" CACHE STRING "" FORCE) + set(CURL_INCLUDE_DIR "${BINARY_DIR}/thirdparty/curl-install/include" CACHE STRING "" FORCE) + set(CURL_INCLUDE_DIRS "${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) + set(CURL_LIBRARY "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" CACHE STRING "" FORCE) + set(CURL_LIBRARIES "${CURL_LIBRARY}" CACHE STRING "" FORCE) - # Set variables - set(CURL_FOUND "YES" CACHE STRING "" FORCE) - set(CURL_INCLUDE_DIR "${BINARY_DIR}/thirdparty/curl-install/include" CACHE STRING "" FORCE) - set(CURL_INCLUDE_DIRS "${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) - set(CURL_LIBRARY "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" CACHE STRING "" FORCE) - set(CURL_LIBRARIES "${CURL_LIBRARY}" CACHE STRING "" FORCE) + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_INCLUDE_DIR=${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_LIBRARY=${CURL_LIBRARY}" CACHE STRING "" FORCE) - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_INCLUDE_DIR=${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_LIBRARY=${CURL_LIBRARY}" CACHE STRING "" FORCE) + # Create imported targets + file(MAKE_DIRECTORY ${CURL_INCLUDE_DIRS}) - # Create imported targets - file(MAKE_DIRECTORY ${CURL_INCLUDE_DIRS}) + add_library(CURL::libcurl STATIC IMPORTED) + set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") + add_dependencies(CURL::libcurl curl-external) + target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) + target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads OpenSSL::SSL OpenSSL::Crypto) + if (APPLE) + target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") + target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") + target_link_libraries(CURL::libcurl INTERFACE "-framework CoreServices") + endif() - add_library(CURL::libcurl STATIC IMPORTED) - set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") - add_dependencies(CURL::libcurl curl-external) - target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) - target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads OpenSSL::SSL OpenSSL::Crypto) - if (APPLE) - target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") - target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") - target_link_libraries(CURL::libcurl INTERFACE "-framework CoreServices") endif() endfunction(use_bundled_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/BundledOpenSSL.cmake b/cmake/BundledOpenSSL.cmake index 275e9ca1dc..0dd272343b 100644 --- a/cmake/BundledOpenSSL.cmake +++ b/cmake/BundledOpenSSL.cmake @@ -18,134 +18,143 @@ function(use_openssl SOURCE_DIR BINARY_DIR) message("Using bundled OpenSSL") - if(APPLE OR WIN32 OR CMAKE_SIZEOF_VOID_P EQUAL 4 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(armv8)") - set(LIBDIR "lib") - else() - set(LIBDIR "lib64") - endif() + if(USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") + include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) - # Define byproducts - set(BYPRODUCT_PREFIX "lib" CACHE STRING "" FORCE) - set(OPENSSL_BUILD_SHARED "NO" CACHE STRING "" FORCE) - - if (WIN32) - set(BYPRODUCT_SUFFIX ".lib" CACHE STRING "" FORCE) - # Due to OpenSSL 3's static linking issue on x86 MacOS platform we make an exception to build a shared library instead - elseif (APPLE AND (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")) - set(BYPRODUCT_SUFFIX ".dylib" CACHE STRING "" FORCE) - set(OPENSSL_BUILD_SHARED "YES" CACHE STRING "" FORCE) - else() - set(BYPRODUCT_SUFFIX ".a" CACHE STRING "" FORCE) - endif() + elseif(USE_STANDALONE_CMAKE) + message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") - set(BYPRODUCTS - "${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" - "${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" - ) + if(APPLE OR WIN32 OR CMAKE_SIZEOF_VOID_P EQUAL 4 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(armv8)") + set(LIBDIR "lib") + else() + set(LIBDIR "lib64") + endif() - if (OPENSSL_BUILD_SHARED) - set(OPENSSL_SHARED_FLAG "" CACHE STRING "" FORCE) - else() - set(OPENSSL_SHARED_FLAG "no-shared" CACHE STRING "" FORCE) - endif() + # Define byproducts + set(BYPRODUCT_PREFIX "lib" CACHE STRING "" FORCE) + set(OPENSSL_BUILD_SHARED "NO" CACHE STRING "" FORCE) - set(OPENSSL_BIN_DIR "${BINARY_DIR}/thirdparty/openssl-install" CACHE STRING "" FORCE) + if (WIN32) + set(BYPRODUCT_SUFFIX ".lib" CACHE STRING "" FORCE) + # Due to OpenSSL 3's static linking issue on x86 MacOS platform we make an exception to build a shared library instead + elseif (APPLE AND (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")) + set(BYPRODUCT_SUFFIX ".dylib" CACHE STRING "" FORCE) + set(OPENSSL_BUILD_SHARED "YES" CACHE STRING "" FORCE) + else() + set(BYPRODUCT_SUFFIX ".a" CACHE STRING "" FORCE) + endif() - FOREACH(BYPRODUCT ${BYPRODUCTS}) - LIST(APPEND OPENSSL_LIBRARIES_LIST "${OPENSSL_BIN_DIR}/${BYPRODUCT}") - ENDFOREACH(BYPRODUCT) + set(BYPRODUCTS + "${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" + "${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" + ) - if (OPENSSL_BUILD_SHARED) - install(FILES ${OPENSSL_LIBRARIES_LIST} DESTINATION bin COMPONENT bin) - endif() + if (OPENSSL_BUILD_SHARED) + set(OPENSSL_SHARED_FLAG "" CACHE STRING "" FORCE) + else() + set(OPENSSL_SHARED_FLAG "no-shared" CACHE STRING "" FORCE) + endif() - # Set build options - set(OPENSSL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${OPENSSL_BIN_DIR}" - "-DCMAKE_POLICY_DEFAULT_CMP0063=NEW" - # avoid polluting the global namespace, otherwise could interfere with the system openssl (e.g. python script extension using numpy) - "-DCMAKE_C_VISIBILITY_PRESET=hidden" - "-DCMAKE_CXX_VISIBILITY_PRESET=hidden" - "-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON" - ) + set(OPENSSL_BIN_DIR "${BINARY_DIR}/thirdparty/openssl-install" CACHE STRING "" FORCE) - # Note: when upgrading to a later release than 3.1.1 the --no-apps could be used instead of --no-tests to minimize the build size - if (WIN32) - find_program(JOM_EXECUTABLE_PATH - NAMES jom.exe - PATHS ENV PATH - NO_DEFAULT_PATH) - if(JOM_EXECUTABLE_PATH) - include(ProcessorCount) - processorcount(jobs) - set(OPENSSL_BUILD_COMMAND ${JOM_EXECUTABLE_PATH} -j${jobs}) - set(OPENSSL_WINDOWS_COMPILE_FLAGS /FS) - else() - message("Using nmake for OpenSSL build") - set(OPENSSL_BUILD_COMMAND nmake) - set(OPENSSL_WINDOWS_COMPILE_FLAGS "") + FOREACH(BYPRODUCT ${BYPRODUCTS}) + LIST(APPEND OPENSSL_LIBRARIES_LIST "${OPENSSL_BIN_DIR}/${BYPRODUCT}") + ENDFOREACH(BYPRODUCT) + + if (OPENSSL_BUILD_SHARED) + install(FILES ${OPENSSL_LIBRARIES_LIST} DESTINATION bin COMPONENT bin) endif() - ExternalProject_Add( - openssl-external - URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz - URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" - SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" - BUILD_IN_SOURCE true - CONFIGURE_COMMAND perl Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" - BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} - EXCLUDE_FROM_ALL TRUE - BUILD_COMMAND ${OPENSSL_BUILD_COMMAND} - INSTALL_COMMAND nmake install + + # Set build options + set(OPENSSL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${OPENSSL_BIN_DIR}" + "-DCMAKE_POLICY_DEFAULT_CMP0063=NEW" + # avoid polluting the global namespace, otherwise could interfere with the system openssl (e.g. python script extension using numpy) + "-DCMAKE_C_VISIBILITY_PRESET=hidden" + "-DCMAKE_CXX_VISIBILITY_PRESET=hidden" + "-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON" + ) + + # Note: when upgrading to a later release than 3.1.1 the --no-apps could be used instead of --no-tests to minimize the build size + if (WIN32) + find_program(JOM_EXECUTABLE_PATH + NAMES jom.exe + PATHS ENV PATH + NO_DEFAULT_PATH) + if(JOM_EXECUTABLE_PATH) + include(ProcessorCount) + processorcount(jobs) + set(OPENSSL_BUILD_COMMAND ${JOM_EXECUTABLE_PATH} -j${jobs}) + set(OPENSSL_WINDOWS_COMPILE_FLAGS /FS) + else() + message("Using nmake for OpenSSL build") + set(OPENSSL_BUILD_COMMAND nmake) + set(OPENSSL_WINDOWS_COMPILE_FLAGS "") + endif() + ExternalProject_Add( + openssl-external + URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz + URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" + SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" + BUILD_IN_SOURCE true + CONFIGURE_COMMAND perl Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" + BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} + EXCLUDE_FROM_ALL TRUE + BUILD_COMMAND ${OPENSSL_BUILD_COMMAND} + INSTALL_COMMAND nmake install + ) + else() + ExternalProject_Add( + openssl-external + URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz + URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" + SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" + BUILD_IN_SOURCE true + CONFIGURE_COMMAND ./Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} -fPIC" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} -fPIC" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" + BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} + EXCLUDE_FROM_ALL TRUE ) - else() - ExternalProject_Add( - openssl-external - URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz - URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" - SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" - BUILD_IN_SOURCE true - CONFIGURE_COMMAND ./Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} -fPIC" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} -fPIC" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" - BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} - EXCLUDE_FROM_ALL TRUE - ) - endif() + endif() + + # Set variables + set(OPENSSL_FOUND "YES" CACHE STRING "" FORCE) + set(OPENSSL_INCLUDE_DIR "${OPENSSL_BIN_DIR}/include" CACHE STRING "" FORCE) + set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES_LIST};${CMAKE_DL_LIBS}" CACHE STRING "" FORCE) + set(OPENSSL_CRYPTO_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) + set(OPENSSL_SSL_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) + + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}" CACHE STRING "" FORCE) + string(REPLACE ";" "%" OPENSSL_LIBRARIES_EXPORT "${OPENSSL_LIBRARIES}") + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_LIBRARIES=${OPENSSL_LIBRARIES_EXPORT}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}" CACHE STRING "" FORCE) + + # Create imported targets + file(MAKE_DIRECTORY ${OPENSSL_INCLUDE_DIR}) + + add_library(OpenSSL::Crypto STATIC IMPORTED) + set_target_properties(OpenSSL::Crypto PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") + set_target_properties(OpenSSL::Crypto PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}") + add_dependencies(OpenSSL::Crypto openssl-external) + + add_library(OpenSSL::SSL STATIC IMPORTED) + set_target_properties(OpenSSL::SSL PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") + set_target_properties(OpenSSL::SSL PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}") + add_dependencies(OpenSSL::SSL openssl-external) + set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) + + if(WIN32) + set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) + set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) + endif() - # Set variables - set(OPENSSL_FOUND "YES" CACHE STRING "" FORCE) - set(OPENSSL_INCLUDE_DIR "${OPENSSL_BIN_DIR}/include" CACHE STRING "" FORCE) - set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES_LIST};${CMAKE_DL_LIBS}" CACHE STRING "" FORCE) - set(OPENSSL_CRYPTO_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) - set(OPENSSL_SSL_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) - - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}" CACHE STRING "" FORCE) - string(REPLACE ";" "%" OPENSSL_LIBRARIES_EXPORT "${OPENSSL_LIBRARIES}") - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_LIBRARIES=${OPENSSL_LIBRARIES_EXPORT}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}" CACHE STRING "" FORCE) - - # Create imported targets - file(MAKE_DIRECTORY ${OPENSSL_INCLUDE_DIR}) - - add_library(OpenSSL::Crypto STATIC IMPORTED) - set_target_properties(OpenSSL::Crypto PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}") - add_dependencies(OpenSSL::Crypto openssl-external) - - add_library(OpenSSL::SSL STATIC IMPORTED) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}") - add_dependencies(OpenSSL::SSL openssl-external) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) - - if(WIN32) - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) endif() endfunction(use_openssl) diff --git a/cmake/BundledZLIB.cmake b/cmake/BundledZLIB.cmake index 16b77c9077..dfea952708 100644 --- a/cmake/BundledZLIB.cmake +++ b/cmake/BundledZLIB.cmake @@ -18,56 +18,65 @@ function(use_bundled_zlib SOURCE_DIR BINARY_DIR) message("Using bundled zlib") - # Define byproducts - if (WIN32) - string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "lib/zlibstatic.lib") + if(USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt zlib external lib") + include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) + + elseif(USE_STANDALONE_CMAKE) + message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") + + # Define byproducts + if (WIN32) + string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "lib/zlibstatic.lib") + else() + set(BYPRODUCT "lib/zlibstaticd.lib") + endif() else() - set(BYPRODUCT "lib/zlibstaticd.lib") + set(BYPRODUCT "lib/libz.a") endif() - else() - set(BYPRODUCT "lib/libz.a") - endif() - # Set build options - set(ZLIB_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/zlib-install" - ) + # Set build options + set(ZLIB_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/zlib-install" + ) - # Build project - ExternalProject_Add( - zlib-external - URL "https://github.com/madler/zlib/archive/v1.2.11.tar.gz" - URL_HASH "SHA256=629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff" - SOURCE_DIR "${BINARY_DIR}/thirdparty/zlib-src" - CMAKE_ARGS ${ZLIB_CMAKE_ARGS} - BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" - EXCLUDE_FROM_ALL TRUE - ) + # Build project + ExternalProject_Add( + zlib-external + URL "https://github.com/madler/zlib/archive/v1.2.11.tar.gz" + URL_HASH "SHA256=629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff" + SOURCE_DIR "${BINARY_DIR}/thirdparty/zlib-src" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS} + BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" + EXCLUDE_FROM_ALL TRUE + ) - # Set variables - set(ZLIB_FOUND "YES" CACHE STRING "" FORCE) - set(ZLIB_INCLUDE_DIRS "${BINARY_DIR}/thirdparty/zlib-install/include" CACHE STRING "" FORCE) - set(ZLIB_LIBRARIES "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" CACHE STRING "" FORCE) - set(ZLIB_VERSION_STRING "1.2.11" CACHE STRING "" FORCE) - set(ZLIB_VERSION_MAJOR 1 CACHE STRING "" FORCE) - set(ZLIB_VERSION_MINOR 2 CACHE STRING "" FORCE) - set(ZLIB_VERSION_PATCH 11 CACHE STRING "" FORCE) + # Set variables + set(ZLIB_FOUND "YES" CACHE STRING "" FORCE) + set(ZLIB_INCLUDE_DIRS "${BINARY_DIR}/thirdparty/zlib-install/include" CACHE STRING "" FORCE) + set(ZLIB_LIBRARIES "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" CACHE STRING "" FORCE) + set(ZLIB_VERSION_STRING "1.2.11" CACHE STRING "" FORCE) + set(ZLIB_VERSION_MAJOR 1 CACHE STRING "" FORCE) + set(ZLIB_VERSION_MINOR 2 CACHE STRING "" FORCE) + set(ZLIB_VERSION_PATCH 11 CACHE STRING "" FORCE) - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_LIBRARIES=${ZLIB_LIBRARIES}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_STRING=${ZLIB_VERSION_STRING}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MAJOR=${ZLIB_VERSION_MAJOR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MINOR=${ZLIB_VERSION_MINOR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_PATCH=${ZLIB_VERSION_PATCH}" CACHE STRING "" FORCE) + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_LIBRARIES=${ZLIB_LIBRARIES}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_STRING=${ZLIB_VERSION_STRING}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MAJOR=${ZLIB_VERSION_MAJOR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MINOR=${ZLIB_VERSION_MINOR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_PATCH=${ZLIB_VERSION_PATCH}" CACHE STRING "" FORCE) - # Create imported targets - file(MAKE_DIRECTORY ${ZLIB_INCLUDE_DIRS}) + # Create imported targets + file(MAKE_DIRECTORY ${ZLIB_INCLUDE_DIRS}) - add_library(ZLIB::ZLIB STATIC IMPORTED) - set_target_properties(ZLIB::ZLIB PROPERTIES IMPORTED_LOCATION "${ZLIB_LIBRARIES}") - add_dependencies(ZLIB::ZLIB zlib-external) - set_property(TARGET ZLIB::ZLIB APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) + add_library(ZLIB::ZLIB STATIC IMPORTED) + set_target_properties(ZLIB::ZLIB PROPERTIES IMPORTED_LOCATION "${ZLIB_LIBRARIES}") + add_dependencies(ZLIB::ZLIB zlib-external) + set_property(TARGET ZLIB::ZLIB APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) + + endif() endfunction(use_bundled_zlib) diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000000..1c33a993ce --- /dev/null +++ b/conanfile.py @@ -0,0 +1,74 @@ +from conan import ConanFile +from conan.tools.env import VirtualRunEnv +from conan.tools.cmake import CMake, CMakeToolchain + +required_conan_version = ">=2.1.0" + +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0") + +class MiNiFiCppMain(ConanFile): + name = "minifi-cpp-main" + version = "0.99.0" + license = "Apache-2.0" + requires = shared_requires + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeDeps" + options = {"shared": [True, False], "fPIC": [True, False]} + + default_options = {"shared": False, "fPIC": True,} + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["USE_CONAN_PACKAGER"] = "ON" + tc.variables["USE_STANDALONE_CMAKE"] = "OFF" + + tc.variables["ENABLE_OPENWSMAN"] = "OFF" + tc.variables["ENABLE_CIVET"] = "OFF" + tc.variables["SKIP_TESTS"] = "ON" + tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "OFF" + tc.variables["ENABLE_BZIP2"] = "OFF" + tc.variables["ENABLE_ROCKSDB"] = "OFF" + tc.variables["BUILD_ROCKSDB"] = "OFF" + + if self.settings.os == "Windows": + tc.variables["ENABLE_WEL"] = "OFF" + tc.variables["ENABLE_PDH"] = "OFF" + tc.variables["ENABLE_SMB"] = "OFF" + elif self.settings.os == "Linux": + tc.variables["ENABLE_SYSTEMD"] = "OFF" + tc.variables["ENABLE_PROCFS"] = "OFF" + + tc.variables["ENABLE_LIBARCHIVE"] = "OFF" + tc.variables["ENABLE_LZMA"] = "OFF" + tc.variables["ENABLE_GPS"] = "OFF" + tc.variables["ENABLE_COAP"] = "OFF" + tc.variables["ENABLE_SQL"] = "OFF" + tc.variables["ENABLE_MQTT"] = "OFF" + tc.variables["ENABLE_PCAP"] = "OFF" + tc.variables["ENABLE_LIBRDKAFKA"] = "OFF" + tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" + tc.variables["ENABLE_PYTHON_SCRIPTING"] = "OFF" + tc.variables["ENABLE_SENSORS"] = "OFF" + tc.variables["ENABLE_USB_CAMERA"] = "OFF" + tc.variables["ENABLE_AWS"] = "OFF" + tc.variables["ENABLE_OPENCV"] = "OFF" + tc.variables["ENABLE_BUSTACHE"] = "OFF" + tc.variables["ENABLE_SFTP"] = "OFF" + tc.variables["ENABLE_AZURE"] = "OFF" + tc.variables["ENABLE_ENCRYPT_CONFIG"] = "OFF" + tc.variables["ENABLE_SPLUNK"] = "OFF" + tc.variables["ENABLE_ELASTICSEARCH"] = "OFF" + tc.variables["ENABLE_GCP"] = "OFF" + tc.variables["ENABLE_KUBERNETES"] = "OFF" + tc.variables["ENABLE_TEST_PROCESSORS"] = "OFF" + tc.variables["ENABLE_PROMETHEUS"] = "OFF" + tc.variables["ENABLE_GRAFANA_LOKI"] = "OFF" + tc.variables["ENABLE_GRPC_FOR_LOKI"] = "OFF" + tc.variables["ENABLE_CONTROLLER"] = "OFF" + + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() From 8457e8849d95fac67d5951243fd8b57c9b7d7582 Mon Sep 17 00:00:00 2001 From: james94 Date: Tue, 21 May 2024 00:16:55 -0700 Subject: [PATCH 02/26] Added USE_CONAN_PACKAGER & USE_STANDALONE_CMAKE to MiNiFiOptions Verified I can also build MiNiFi using standalone CMake still --- cmake/MiNiFiOptions.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 536e93557a..203eb2b5e9 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -42,6 +42,8 @@ add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets add_minifi_option(DOCKER_PUSH "Push created images to the specified tags" OFF) add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specific optimizations" ON) +add_minifi_option(USE_CONAN_PACKAGER "Instructs cmake to use conan packager to install external libs & build MiNiFi C++" OFF) +add_minifi_option(USE_STANDALONE_CMAKE "Instructs cmake to use only cmake to install external libs & build MiNiFi C++" ON) add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON) add_minifi_dependent_option(STATIC_BUILD "Attempts to statically link as many dependencies as possible." ON "NOT USE_SHARED_LIBS" OFF) add_minifi_option(LIBC_STATIC "Instructs the build system to statically link libstdc++ and glibc into minifiexe. Experiemental" OFF) From ff0e1598fca21a06e844b6e10dc33540314282b7 Mon Sep 17 00:00:00 2001 From: james94 Date: Tue, 21 May 2024 20:45:18 -0700 Subject: [PATCH 03/26] Review update: Added Get LibCURL, OpenSSL, ZLIB cmake & built MiNiFi Addressed martinzink's feedback adding GetLibCURL, GetOpenSSL and GetZLIB cmake files that use prebuilt conan package when MINIFI_USE_CONAN_PACKAGER is set, otherwise it uses bundled cmake files from standalone cmake approach. After updating the files, I verified I could build MiNIFi using conan build and then verified I could build MiNiFi using standalone CMake approach. Furthermore, I also verified I could run minifi binary --- CMakeLists.txt | 12 +- cmake/BundledLibcURL.cmake | 146 ++++++++++------------ cmake/BundledOpenSSL.cmake | 246 ++++++++++++++++++------------------- cmake/BundledZLIB.cmake | 97 +++++++-------- cmake/GetLibCURL.cmake | 27 ++++ cmake/GetOpenSSL.cmake | 27 ++++ cmake/GetZLIB.cmake | 27 ++++ cmake/MiNiFiOptions.cmake | 3 +- 8 files changed, 319 insertions(+), 266 deletions(-) create mode 100644 cmake/GetLibCURL.cmake create mode 100644 cmake/GetOpenSSL.cmake create mode 100644 cmake/GetZLIB.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1699e95f99..78dace17b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,8 +239,8 @@ if(NOT WIN32) endif() # OpenSSL -include(BundledOpenSSL) -use_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +include(GetOpenSSL) +get_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") # libsodium @@ -250,13 +250,13 @@ use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR} list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1) # zlib -include(BundledZLIB) -use_bundled_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include(GetZLIB) +get_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zlib/dummy") # cURL -include(BundledLibcURL) -use_bundled_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include(GetLibCURL) +get_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy") # spdlog diff --git a/cmake/BundledLibcURL.cmake b/cmake/BundledLibcURL.cmake index 8f50c80ea3..bba5f70ab0 100644 --- a/cmake/BundledLibcURL.cmake +++ b/cmake/BundledLibcURL.cmake @@ -16,93 +16,83 @@ # under the License. function(use_bundled_curl SOURCE_DIR BINARY_DIR) - if(USE_CONAN_PACKAGER) - message("Using Conan Packager to manage installing prebuilt libcurl external lib") - include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) - - elseif(USE_STANDALONE_CMAKE) - message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") - - - # Define patch step - set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/curl/module-path.patch") - set(PC ${Bash_EXECUTABLE} -c "set -x && \ - (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\")") - # Define byproducts - string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) - if (WIN32) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "lib/libcurl.lib") - else() - set(BYPRODUCT "lib/libcurl-d.lib") - endif() + # Define patch step + set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/curl/module-path.patch") + set(PC ${Bash_EXECUTABLE} -c "set -x && \ + (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\")") + # Define byproducts + string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) + if (WIN32) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "lib/libcurl.lib") else() - include(GNUInstallDirs) - string(REPLACE "/" ";" LIBDIR_LIST ${CMAKE_INSTALL_LIBDIR}) - list(GET LIBDIR_LIST 0 LIBDIR) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "${LIBDIR}/libcurl.a") - else() - set(BYPRODUCT "${LIBDIR}/libcurl-d.a") - endif() + set(BYPRODUCT "lib/libcurl-d.lib") endif() + else() + include(GNUInstallDirs) + string(REPLACE "/" ";" LIBDIR_LIST ${CMAKE_INSTALL_LIBDIR}) + list(GET LIBDIR_LIST 0 LIBDIR) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "${LIBDIR}/libcurl.a") + else() + set(BYPRODUCT "${LIBDIR}/libcurl-d.a") + endif() + endif() - # Set build options - set(CURL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/curl-install" - -DBUILD_CURL_EXE=OFF - -DBUILD_TESTING=OFF - -DBUILD_SHARED_LIBS=OFF - -DHTTP_ONLY=ON - -DCURL_CA_PATH=none - -DCURL_USE_LIBSSH2=OFF - -DUSE_LIBIDN2=OFF - -DCURL_USE_LIBPSL=OFF - -DCURL_USE_OPENSSL=ON - ) - - append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") + # Set build options + set(CURL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/curl-install" + -DBUILD_CURL_EXE=OFF + -DBUILD_TESTING=OFF + -DBUILD_SHARED_LIBS=OFF + -DHTTP_ONLY=ON + -DCURL_CA_PATH=none + -DCURL_USE_LIBSSH2=OFF + -DUSE_LIBIDN2=OFF + -DCURL_USE_LIBPSL=OFF + -DCURL_USE_OPENSSL=ON + ) - # Build project - ExternalProject_Add( - curl-external - URL "https://github.com/curl/curl/releases/download/curl-8_4_0/curl-8.4.0.tar.gz" - URL_HASH "SHA256=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" - SOURCE_DIR "${BINARY_DIR}/thirdparty/curl-src" - LIST_SEPARATOR % # This is needed for passing semicolon-separated lists - CMAKE_ARGS ${CURL_CMAKE_ARGS} - PATCH_COMMAND ${PC} - BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" - EXCLUDE_FROM_ALL TRUE - ) + append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") - # Set dependencies - add_dependencies(curl-external ZLIB::ZLIB OpenSSL::SSL OpenSSL::Crypto) + # Build project + ExternalProject_Add( + curl-external + URL "https://github.com/curl/curl/releases/download/curl-8_4_0/curl-8.4.0.tar.gz" + URL_HASH "SHA256=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" + SOURCE_DIR "${BINARY_DIR}/thirdparty/curl-src" + LIST_SEPARATOR % # This is needed for passing semicolon-separated lists + CMAKE_ARGS ${CURL_CMAKE_ARGS} + PATCH_COMMAND ${PC} + BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" + EXCLUDE_FROM_ALL TRUE + ) - # Set variables - set(CURL_FOUND "YES" CACHE STRING "" FORCE) - set(CURL_INCLUDE_DIR "${BINARY_DIR}/thirdparty/curl-install/include" CACHE STRING "" FORCE) - set(CURL_INCLUDE_DIRS "${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) - set(CURL_LIBRARY "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" CACHE STRING "" FORCE) - set(CURL_LIBRARIES "${CURL_LIBRARY}" CACHE STRING "" FORCE) + # Set dependencies + add_dependencies(curl-external ZLIB::ZLIB OpenSSL::SSL OpenSSL::Crypto) - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_INCLUDE_DIR=${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_LIBRARY=${CURL_LIBRARY}" CACHE STRING "" FORCE) + # Set variables + set(CURL_FOUND "YES" CACHE STRING "" FORCE) + set(CURL_INCLUDE_DIR "${BINARY_DIR}/thirdparty/curl-install/include" CACHE STRING "" FORCE) + set(CURL_INCLUDE_DIRS "${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) + set(CURL_LIBRARY "${BINARY_DIR}/thirdparty/curl-install/${BYPRODUCT}" CACHE STRING "" FORCE) + set(CURL_LIBRARIES "${CURL_LIBRARY}" CACHE STRING "" FORCE) - # Create imported targets - file(MAKE_DIRECTORY ${CURL_INCLUDE_DIRS}) + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_INCLUDE_DIR=${CURL_INCLUDE_DIR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_CURL_LIBRARY=${CURL_LIBRARY}" CACHE STRING "" FORCE) - add_library(CURL::libcurl STATIC IMPORTED) - set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") - add_dependencies(CURL::libcurl curl-external) - target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) - target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads OpenSSL::SSL OpenSSL::Crypto) - if (APPLE) - target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") - target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") - target_link_libraries(CURL::libcurl INTERFACE "-framework CoreServices") - endif() + # Create imported targets + file(MAKE_DIRECTORY ${CURL_INCLUDE_DIRS}) + add_library(CURL::libcurl STATIC IMPORTED) + set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") + add_dependencies(CURL::libcurl curl-external) + target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) + target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads OpenSSL::SSL OpenSSL::Crypto) + if (APPLE) + target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") + target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") + target_link_libraries(CURL::libcurl INTERFACE "-framework CoreServices") endif() endfunction(use_bundled_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/BundledOpenSSL.cmake b/cmake/BundledOpenSSL.cmake index 0dd272343b..4ffa51e80e 100644 --- a/cmake/BundledOpenSSL.cmake +++ b/cmake/BundledOpenSSL.cmake @@ -18,143 +18,135 @@ function(use_openssl SOURCE_DIR BINARY_DIR) message("Using bundled OpenSSL") - if(USE_CONAN_PACKAGER) - message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") - include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) - - elseif(USE_STANDALONE_CMAKE) - message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") - - if(APPLE OR WIN32 OR CMAKE_SIZEOF_VOID_P EQUAL 4 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(armv8)") - set(LIBDIR "lib") - else() - set(LIBDIR "lib64") - endif() - - # Define byproducts - set(BYPRODUCT_PREFIX "lib" CACHE STRING "" FORCE) - set(OPENSSL_BUILD_SHARED "NO" CACHE STRING "" FORCE) + if(APPLE OR WIN32 OR CMAKE_SIZEOF_VOID_P EQUAL 4 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(arm64)|(ARM64)|(aarch64)|(armv8)") + set(LIBDIR "lib") + else() + set(LIBDIR "lib64") + endif() - if (WIN32) - set(BYPRODUCT_SUFFIX ".lib" CACHE STRING "" FORCE) - # Due to OpenSSL 3's static linking issue on x86 MacOS platform we make an exception to build a shared library instead - elseif (APPLE AND (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")) - set(BYPRODUCT_SUFFIX ".dylib" CACHE STRING "" FORCE) - set(OPENSSL_BUILD_SHARED "YES" CACHE STRING "" FORCE) - else() - set(BYPRODUCT_SUFFIX ".a" CACHE STRING "" FORCE) - endif() + # Define byproducts + set(BYPRODUCT_PREFIX "lib" CACHE STRING "" FORCE) + set(OPENSSL_BUILD_SHARED "NO" CACHE STRING "" FORCE) + + if (WIN32) + set(BYPRODUCT_SUFFIX ".lib" CACHE STRING "" FORCE) + # Due to OpenSSL 3's static linking issue on x86 MacOS platform we make an exception to build a shared library instead + elseif (APPLE AND (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")) + set(BYPRODUCT_SUFFIX ".dylib" CACHE STRING "" FORCE) + set(OPENSSL_BUILD_SHARED "YES" CACHE STRING "" FORCE) + else() + set(BYPRODUCT_SUFFIX ".a" CACHE STRING "" FORCE) + endif() - set(BYPRODUCTS - "${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" - "${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" - ) + set(BYPRODUCTS + "${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" + "${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" + ) - if (OPENSSL_BUILD_SHARED) - set(OPENSSL_SHARED_FLAG "" CACHE STRING "" FORCE) - else() - set(OPENSSL_SHARED_FLAG "no-shared" CACHE STRING "" FORCE) - endif() + if (OPENSSL_BUILD_SHARED) + set(OPENSSL_SHARED_FLAG "" CACHE STRING "" FORCE) + else() + set(OPENSSL_SHARED_FLAG "no-shared" CACHE STRING "" FORCE) + endif() - set(OPENSSL_BIN_DIR "${BINARY_DIR}/thirdparty/openssl-install" CACHE STRING "" FORCE) + set(OPENSSL_BIN_DIR "${BINARY_DIR}/thirdparty/openssl-install" CACHE STRING "" FORCE) - FOREACH(BYPRODUCT ${BYPRODUCTS}) - LIST(APPEND OPENSSL_LIBRARIES_LIST "${OPENSSL_BIN_DIR}/${BYPRODUCT}") - ENDFOREACH(BYPRODUCT) + FOREACH(BYPRODUCT ${BYPRODUCTS}) + LIST(APPEND OPENSSL_LIBRARIES_LIST "${OPENSSL_BIN_DIR}/${BYPRODUCT}") + ENDFOREACH(BYPRODUCT) - if (OPENSSL_BUILD_SHARED) - install(FILES ${OPENSSL_LIBRARIES_LIST} DESTINATION bin COMPONENT bin) - endif() + if (OPENSSL_BUILD_SHARED) + install(FILES ${OPENSSL_LIBRARIES_LIST} DESTINATION bin COMPONENT bin) + endif() - # Set build options - set(OPENSSL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${OPENSSL_BIN_DIR}" - "-DCMAKE_POLICY_DEFAULT_CMP0063=NEW" - # avoid polluting the global namespace, otherwise could interfere with the system openssl (e.g. python script extension using numpy) - "-DCMAKE_C_VISIBILITY_PRESET=hidden" - "-DCMAKE_CXX_VISIBILITY_PRESET=hidden" - "-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON" - ) - - # Note: when upgrading to a later release than 3.1.1 the --no-apps could be used instead of --no-tests to minimize the build size - if (WIN32) - find_program(JOM_EXECUTABLE_PATH - NAMES jom.exe - PATHS ENV PATH - NO_DEFAULT_PATH) - if(JOM_EXECUTABLE_PATH) - include(ProcessorCount) - processorcount(jobs) - set(OPENSSL_BUILD_COMMAND ${JOM_EXECUTABLE_PATH} -j${jobs}) - set(OPENSSL_WINDOWS_COMPILE_FLAGS /FS) - else() - message("Using nmake for OpenSSL build") - set(OPENSSL_BUILD_COMMAND nmake) - set(OPENSSL_WINDOWS_COMPILE_FLAGS "") - endif() - ExternalProject_Add( - openssl-external - URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz - URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" - SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" - BUILD_IN_SOURCE true - CONFIGURE_COMMAND perl Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" - BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} - EXCLUDE_FROM_ALL TRUE - BUILD_COMMAND ${OPENSSL_BUILD_COMMAND} - INSTALL_COMMAND nmake install - ) - else() - ExternalProject_Add( - openssl-external - URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz - URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" - SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" - BUILD_IN_SOURCE true - CONFIGURE_COMMAND ./Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} -fPIC" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} -fPIC" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" - BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} - EXCLUDE_FROM_ALL TRUE + # Set build options + set(OPENSSL_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${OPENSSL_BIN_DIR}" + "-DCMAKE_POLICY_DEFAULT_CMP0063=NEW" + # avoid polluting the global namespace, otherwise could interfere with the system openssl (e.g. python script extension using numpy) + "-DCMAKE_C_VISIBILITY_PRESET=hidden" + "-DCMAKE_CXX_VISIBILITY_PRESET=hidden" + "-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON" ) - endif() - # Set variables - set(OPENSSL_FOUND "YES" CACHE STRING "" FORCE) - set(OPENSSL_INCLUDE_DIR "${OPENSSL_BIN_DIR}/include" CACHE STRING "" FORCE) - set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES_LIST};${CMAKE_DL_LIBS}" CACHE STRING "" FORCE) - set(OPENSSL_CRYPTO_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) - set(OPENSSL_SSL_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) - - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}" CACHE STRING "" FORCE) - string(REPLACE ";" "%" OPENSSL_LIBRARIES_EXPORT "${OPENSSL_LIBRARIES}") - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_LIBRARIES=${OPENSSL_LIBRARIES_EXPORT}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}" CACHE STRING "" FORCE) - - # Create imported targets - file(MAKE_DIRECTORY ${OPENSSL_INCLUDE_DIR}) - - add_library(OpenSSL::Crypto STATIC IMPORTED) - set_target_properties(OpenSSL::Crypto PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}") - add_dependencies(OpenSSL::Crypto openssl-external) - - add_library(OpenSSL::SSL STATIC IMPORTED) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}") - add_dependencies(OpenSSL::SSL openssl-external) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) - - if(WIN32) - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) + # Note: when upgrading to a later release than 3.1.1 the --no-apps could be used instead of --no-tests to minimize the build size + if (WIN32) + find_program(JOM_EXECUTABLE_PATH + NAMES jom.exe + PATHS ENV PATH + NO_DEFAULT_PATH) + if(JOM_EXECUTABLE_PATH) + include(ProcessorCount) + processorcount(jobs) + set(OPENSSL_BUILD_COMMAND ${JOM_EXECUTABLE_PATH} -j${jobs}) + set(OPENSSL_WINDOWS_COMPILE_FLAGS /FS) + else() + message("Using nmake for OpenSSL build") + set(OPENSSL_BUILD_COMMAND nmake) + set(OPENSSL_WINDOWS_COMPILE_FLAGS "") endif() + ExternalProject_Add( + openssl-external + URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz + URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" + SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" + BUILD_IN_SOURCE true + CONFIGURE_COMMAND perl Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} ${OPENSSL_WINDOWS_COMPILE_FLAGS}" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" + BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} + EXCLUDE_FROM_ALL TRUE + BUILD_COMMAND ${OPENSSL_BUILD_COMMAND} + INSTALL_COMMAND nmake install + ) + else() + ExternalProject_Add( + openssl-external + URL https://github.com/openssl/openssl/releases/download/openssl-3.3.2/openssl-3.3.2.tar.gz + URL_HASH "SHA256=2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281" + SOURCE_DIR "${BINARY_DIR}/thirdparty/openssl-src" + BUILD_IN_SOURCE true + CONFIGURE_COMMAND ./Configure "CFLAGS=${PASSTHROUGH_CMAKE_C_FLAGS} -fPIC" "CXXFLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS} -fPIC" ${OPENSSL_SHARED_FLAG} no-tests "--prefix=${OPENSSL_BIN_DIR}" "--openssldir=${OPENSSL_BIN_DIR}" + BUILD_BYPRODUCTS ${OPENSSL_LIBRARIES_LIST} + EXCLUDE_FROM_ALL TRUE + ) + endif() + # Set variables + set(OPENSSL_FOUND "YES" CACHE STRING "" FORCE) + set(OPENSSL_INCLUDE_DIR "${OPENSSL_BIN_DIR}/include" CACHE STRING "" FORCE) + set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES_LIST};${CMAKE_DL_LIBS}" CACHE STRING "" FORCE) + set(OPENSSL_CRYPTO_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) + set(OPENSSL_SSL_LIBRARY "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}" CACHE STRING "" FORCE) + + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}" CACHE STRING "" FORCE) + string(REPLACE ";" "%" OPENSSL_LIBRARIES_EXPORT "${OPENSSL_LIBRARIES}") + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_LIBRARIES=${OPENSSL_LIBRARIES_EXPORT}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_OPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}" CACHE STRING "" FORCE) + + # Create imported targets + file(MAKE_DIRECTORY ${OPENSSL_INCLUDE_DIR}) + + add_library(OpenSSL::Crypto STATIC IMPORTED) + set_target_properties(OpenSSL::Crypto PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") + set_target_properties(OpenSSL::Crypto PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}crypto${BYPRODUCT_SUFFIX}") + add_dependencies(OpenSSL::Crypto openssl-external) + + add_library(OpenSSL::SSL STATIC IMPORTED) + set_target_properties(OpenSSL::SSL PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") + set_target_properties(OpenSSL::SSL PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${OPENSSL_BIN_DIR}/${LIBDIR}/${BYPRODUCT_PREFIX}ssl${BYPRODUCT_SUFFIX}") + add_dependencies(OpenSSL::SSL openssl-external) + set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) + + if(WIN32) + set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) + set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) endif() + endfunction(use_openssl) diff --git a/cmake/BundledZLIB.cmake b/cmake/BundledZLIB.cmake index dfea952708..16b77c9077 100644 --- a/cmake/BundledZLIB.cmake +++ b/cmake/BundledZLIB.cmake @@ -18,65 +18,56 @@ function(use_bundled_zlib SOURCE_DIR BINARY_DIR) message("Using bundled zlib") - if(USE_CONAN_PACKAGER) - message("Using Conan Packager to manage installing prebuilt zlib external lib") - include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) - - elseif(USE_STANDALONE_CMAKE) - message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") - - # Define byproducts - if (WIN32) - string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) - if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) - set(BYPRODUCT "lib/zlibstatic.lib") - else() - set(BYPRODUCT "lib/zlibstaticd.lib") - endif() + # Define byproducts + if (WIN32) + string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) + if (build_type MATCHES relwithdebinfo OR build_type MATCHES release) + set(BYPRODUCT "lib/zlibstatic.lib") else() - set(BYPRODUCT "lib/libz.a") + set(BYPRODUCT "lib/zlibstaticd.lib") endif() + else() + set(BYPRODUCT "lib/libz.a") + endif() - # Set build options - set(ZLIB_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/zlib-install" - ) - - # Build project - ExternalProject_Add( - zlib-external - URL "https://github.com/madler/zlib/archive/v1.2.11.tar.gz" - URL_HASH "SHA256=629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff" - SOURCE_DIR "${BINARY_DIR}/thirdparty/zlib-src" - CMAKE_ARGS ${ZLIB_CMAKE_ARGS} - BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" - EXCLUDE_FROM_ALL TRUE - ) + # Set build options + set(ZLIB_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/zlib-install" + ) - # Set variables - set(ZLIB_FOUND "YES" CACHE STRING "" FORCE) - set(ZLIB_INCLUDE_DIRS "${BINARY_DIR}/thirdparty/zlib-install/include" CACHE STRING "" FORCE) - set(ZLIB_LIBRARIES "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" CACHE STRING "" FORCE) - set(ZLIB_VERSION_STRING "1.2.11" CACHE STRING "" FORCE) - set(ZLIB_VERSION_MAJOR 1 CACHE STRING "" FORCE) - set(ZLIB_VERSION_MINOR 2 CACHE STRING "" FORCE) - set(ZLIB_VERSION_PATCH 11 CACHE STRING "" FORCE) + # Build project + ExternalProject_Add( + zlib-external + URL "https://github.com/madler/zlib/archive/v1.2.11.tar.gz" + URL_HASH "SHA256=629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff" + SOURCE_DIR "${BINARY_DIR}/thirdparty/zlib-src" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS} + BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" + EXCLUDE_FROM_ALL TRUE + ) - # Set exported variables for FindPackage.cmake - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_LIBRARIES=${ZLIB_LIBRARIES}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_STRING=${ZLIB_VERSION_STRING}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MAJOR=${ZLIB_VERSION_MAJOR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MINOR=${ZLIB_VERSION_MINOR}" CACHE STRING "" FORCE) - set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_PATCH=${ZLIB_VERSION_PATCH}" CACHE STRING "" FORCE) + # Set variables + set(ZLIB_FOUND "YES" CACHE STRING "" FORCE) + set(ZLIB_INCLUDE_DIRS "${BINARY_DIR}/thirdparty/zlib-install/include" CACHE STRING "" FORCE) + set(ZLIB_LIBRARIES "${BINARY_DIR}/thirdparty/zlib-install/${BYPRODUCT}" CACHE STRING "" FORCE) + set(ZLIB_VERSION_STRING "1.2.11" CACHE STRING "" FORCE) + set(ZLIB_VERSION_MAJOR 1 CACHE STRING "" FORCE) + set(ZLIB_VERSION_MINOR 2 CACHE STRING "" FORCE) + set(ZLIB_VERSION_PATCH 11 CACHE STRING "" FORCE) - # Create imported targets - file(MAKE_DIRECTORY ${ZLIB_INCLUDE_DIRS}) + # Set exported variables for FindPackage.cmake + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_LIBRARIES=${ZLIB_LIBRARIES}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_STRING=${ZLIB_VERSION_STRING}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MAJOR=${ZLIB_VERSION_MAJOR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_MINOR=${ZLIB_VERSION_MINOR}" CACHE STRING "" FORCE) + set(PASSTHROUGH_VARIABLES ${PASSTHROUGH_VARIABLES} "-DEXPORTED_ZLIB_VERSION_PATCH=${ZLIB_VERSION_PATCH}" CACHE STRING "" FORCE) - add_library(ZLIB::ZLIB STATIC IMPORTED) - set_target_properties(ZLIB::ZLIB PROPERTIES IMPORTED_LOCATION "${ZLIB_LIBRARIES}") - add_dependencies(ZLIB::ZLIB zlib-external) - set_property(TARGET ZLIB::ZLIB APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) + # Create imported targets + file(MAKE_DIRECTORY ${ZLIB_INCLUDE_DIRS}) - endif() + add_library(ZLIB::ZLIB STATIC IMPORTED) + set_target_properties(ZLIB::ZLIB PROPERTIES IMPORTED_LOCATION "${ZLIB_LIBRARIES}") + add_dependencies(ZLIB::ZLIB zlib-external) + set_property(TARGET ZLIB::ZLIB APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIRS}) endfunction(use_bundled_zlib) diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake new file mode 100644 index 0000000000..3809960043 --- /dev/null +++ b/cmake/GetLibCURL.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_curl SOURCE_DIR BINARY_DIR) + if(MINIFI_USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt libcurl external lib") + include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) + else() + message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") + include(BundledLibcURL) + use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake new file mode 100644 index 0000000000..ac4b8ddc1d --- /dev/null +++ b/cmake/GetOpenSSL.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_openssl SOURCE_DIR BINARY_DIR) + if(MINIFI_USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") + include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) + else() + message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") + include(BundledOpenSSL) + use_openssl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_openssl) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake new file mode 100644 index 0000000000..efa57b2d08 --- /dev/null +++ b/cmake/GetZLIB.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_zlib SOURCE_DIR BINARY_DIR) + if(MINIFI_USE_CONAN_PACKAGER) + message("Using Conan Packager to manage installing prebuilt zlib external lib") + include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) + else() + message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") + include(BundledZLIB) + use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_zlib) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 203eb2b5e9..458acc957c 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -42,8 +42,7 @@ add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets add_minifi_option(DOCKER_PUSH "Push created images to the specified tags" OFF) add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specific optimizations" ON) -add_minifi_option(USE_CONAN_PACKAGER "Instructs cmake to use conan packager to install external libs & build MiNiFi C++" OFF) -add_minifi_option(USE_STANDALONE_CMAKE "Instructs cmake to use only cmake to install external libs & build MiNiFi C++" ON) +add_minifi_option(MINIFI_USE_CONAN_PACKAGER "Instructs cmake to use conan packager to install external libs & build MiNiFi C++" OFF) add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON) add_minifi_dependent_option(STATIC_BUILD "Attempts to statically link as many dependencies as possible." ON "NOT USE_SHARED_LIBS" OFF) add_minifi_option(LIBC_STATIC "Instructs the build system to statically link libstdc++ and glibc into minifiexe. Experiemental" OFF) From c5ff35fa11d18b7d4dc4ea03843c3c3e49e29a39 Mon Sep 17 00:00:00 2001 From: james94 Date: Wed, 5 Jun 2024 15:35:55 -0700 Subject: [PATCH 04/26] Removed USE_STANDALONE_CMAKE conan cmake tc variable, only need USE_CONAN_PACKAGER tc variables can be used to change minifi options --- conanfile.py | 1 - etc/conan/profiles/release-linux | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 etc/conan/profiles/release-linux diff --git a/conanfile.py b/conanfile.py index 1c33a993ce..61625cbdc9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -20,7 +20,6 @@ class MiNiFiCppMain(ConanFile): def generate(self): tc = CMakeToolchain(self) tc.variables["USE_CONAN_PACKAGER"] = "ON" - tc.variables["USE_STANDALONE_CMAKE"] = "OFF" tc.variables["ENABLE_OPENWSMAN"] = "OFF" tc.variables["ENABLE_CIVET"] = "OFF" diff --git a/etc/conan/profiles/release-linux b/etc/conan/profiles/release-linux new file mode 100644 index 0000000000..26b4fa6a08 --- /dev/null +++ b/etc/conan/profiles/release-linux @@ -0,0 +1,12 @@ +[settings] +os=Linux +arch=x86_64 +build_type=Release +compiler=gcc +compiler.cppstd=gnu20 +compiler.libcxx=libstdc++11 +compiler.version=11 +[conf] +tools.system.package_manager:mode=install +tools.system.package_manager:sudo=True +[options] From 6275b7683e38cc7d675e6a78a9b009058f41f495 Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 8 Jun 2024 12:18:10 -0700 Subject: [PATCH 05/26] Updated conanfile.py tc.variables to MINIFI_USE_CONAN_PACKAGER After addressing some feedback and working with Marton on some build issues he was facing with conan on his side, I realized that conan's OpenSSL, ZLib and LibCURL were being ignored because I didn't update conanfile.py file's USE_CONAN_PACKAGER to be MINIFI_USE_CONAN_PACKAGER to match with MiNiFiOptions.cmake one. Once I made that I update then when building MiNiFi C++, it shows in the build output for OpenSSL, 'Using Conan Packager to manage installing prebuilt OpenSSL external lib' and similar messages for zlib and libcurl. Thus, OpenSSL, ZLib and LibCURL arent build from source since we use the conan prebuilt ones. Also I verified that I can build MiNiFi C++ using default conan v2 profile using the similar steps Marton tried. The main reason I suggested we use a particular conan profile in MiNiFi C++ is so each environment where MiNiFi C++ is built has a consistent conan profile since each environment default generated conan profile could be unique to that environment, which could cause issues in downloading prebuilt conan packages. --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 61625cbdc9..cd08cdb121 100644 --- a/conanfile.py +++ b/conanfile.py @@ -19,7 +19,7 @@ class MiNiFiCppMain(ConanFile): def generate(self): tc = CMakeToolchain(self) - tc.variables["USE_CONAN_PACKAGER"] = "ON" + tc.variables["MINIFI_USE_CONAN_PACKAGER"] = "ON" tc.variables["ENABLE_OPENWSMAN"] = "OFF" tc.variables["ENABLE_CIVET"] = "OFF" From 6312cf008c4a4167241b39765ae3fe8f2ccc73c7 Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 8 Jun 2024 14:58:33 -0700 Subject: [PATCH 06/26] Replaced MINIFI_USE_CONAN_PACKAGER w MINIFI_{ZLIB,OPENSSL,LIBCURL}_SOURCE vars Was able to build MiNiFi C++ with the ENABLE_{External_Lib} variables set to OFF initially. However, after a suggestion from Marton to leave the ENABLE external lib variables set to their default values from MiNiFiOptions.cmake, I removed the tc.variables['ENABLE_{...}'] that overrode MiNiFiOptions.cmake ones and now am trying to build MiNiFi C++ using conan where we use conan's OpenSSL, ZLib and LibCURL prebuilt conan packages alternative to building them from source. I am debugging issues that come up from building MiNiFi using conan where partly we use conan's prebuilt conan packages and then the remaining external libs are the ones built from source. For instance, one finding related to needing to use the predefined conan profile located in etc/conan/profiles/release-linux because of the gnu20 was required to build rocksdb successfully. I found that for my environment, conan's generated default profile used gnu17, which caused rocksdb to fail building, but setting conan to use gnu20 allowed it to succeed in building rocksdb. --- CMakeLists.txt | 11 +++++---- cmake/GetLibCURL.cmake | 4 ++-- cmake/GetOpenSSL.cmake | 7 +++--- cmake/GetZLIB.cmake | 4 ++-- cmake/MiNiFiOptions.cmake | 33 ++++++++++++++++++-------- conanfile.py | 49 +++------------------------------------ 6 files changed, 40 insertions(+), 68 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78dace17b1..e3f5e7c63a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,6 +221,12 @@ else() message(VERBOSE "No custom malloc implementation") endif() +# OpenSSL +include(GetOpenSSL) +get_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") + + if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) include(BundledBZip2) use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) @@ -238,11 +244,6 @@ if(NOT WIN32) use_bundled_osspuuid(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) endif() -# OpenSSL -include(GetOpenSSL) -get_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") - # libsodium include(BundledLibSodium) use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index 3809960043..6fb39c00c1 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -16,10 +16,10 @@ # under the License. function(get_curl SOURCE_DIR BINARY_DIR) - if(MINIFI_USE_CONAN_PACKAGER) + if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt libcurl external lib") include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) - else() + elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") include(BundledLibcURL) use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index ac4b8ddc1d..ab1d7bb3bd 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -16,10 +16,11 @@ # under the License. function(get_openssl SOURCE_DIR BINARY_DIR) - if(MINIFI_USE_CONAN_PACKAGER) + if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") - include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) - else() + # include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) + find_package(OpenSSL REQUIRED) + elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") include(BundledOpenSSL) use_openssl(${SOURCE_DIR} ${BINARY_DIR}) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index efa57b2d08..aae8d59e80 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -16,10 +16,10 @@ # under the License. function(get_zlib SOURCE_DIR BINARY_DIR) - if(MINIFI_USE_CONAN_PACKAGER) + if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt zlib external lib") include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) - else() + elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") include(BundledZLIB) use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 458acc957c..91fdb411cb 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -23,6 +23,13 @@ function(add_minifi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE) set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) endfunction() +function(add_minifi_multi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUES DEFAULT_VALUE) + set(${OPTION_NAME} ${DEFAULT_VALUE} CACHE STRING ${OPTION_DESCRIPTION}) + set_property(CACHE ${OPTION_NAME} PROPERTY STRINGS ${OPTION_VALUES}) + list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) + set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) +endfunction() + function(add_minifi_dependent_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE DEPENDS FORCE) cmake_dependent_option(${OPTION_NAME} ${OPTION_DESCRIPTION} ${OPTION_VALUE} ${DEPENDS} ${FORCE}) list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) @@ -42,7 +49,6 @@ add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets add_minifi_option(DOCKER_PUSH "Push created images to the specified tags" OFF) add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specific optimizations" ON) -add_minifi_option(MINIFI_USE_CONAN_PACKAGER "Instructs cmake to use conan packager to install external libs & build MiNiFi C++" OFF) add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON) add_minifi_dependent_option(STATIC_BUILD "Attempts to statically link as many dependencies as possible." ON "NOT USE_SHARED_LIBS" OFF) add_minifi_option(LIBC_STATIC "Instructs the build system to statically link libstdc++ and glibc into minifiexe. Experiemental" OFF) @@ -66,15 +72,6 @@ add_minifi_option(AWS_ENABLE_UNITY_BUILD "If enabled, AWS SDK libraries will be add_minifi_dependent_option(MINIFI_ADVANCED_ASAN_BUILD "Uses AddressSanitizer to instrument the code" OFF "NOT WIN32" OFF) add_minifi_dependent_option(MINIFI_ADVANCED_CODE_COVERAGE "Use coverage build options and enable coverage build target" OFF "NOT WIN32" OFF) -# Option: STRICT_GSL_CHECKS -# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() -# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) -# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) -# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() -set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") -list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) -set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) - if (WIN32) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_MERGE_MODULES "Include merge modules for Visual C++ Redistributable with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_DLLS "Include Visual C++ Redistributable DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) @@ -124,6 +121,22 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_minifi_option(ENABLE_PROCFS "Enables the procfs extension." ON) endif() +## Each Option provides a list of possible values + +# Option: STRICT_GSL_CHECKS +# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() +# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) +# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) +# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() +set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") +list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) +set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) + +add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") + + # Docker options diff --git a/conanfile.py b/conanfile.py index cd08cdb121..afe42203af 100644 --- a/conanfile.py +++ b/conanfile.py @@ -19,52 +19,9 @@ class MiNiFiCppMain(ConanFile): def generate(self): tc = CMakeToolchain(self) - tc.variables["MINIFI_USE_CONAN_PACKAGER"] = "ON" - - tc.variables["ENABLE_OPENWSMAN"] = "OFF" - tc.variables["ENABLE_CIVET"] = "OFF" - tc.variables["SKIP_TESTS"] = "ON" - tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "OFF" - tc.variables["ENABLE_BZIP2"] = "OFF" - tc.variables["ENABLE_ROCKSDB"] = "OFF" - tc.variables["BUILD_ROCKSDB"] = "OFF" - - if self.settings.os == "Windows": - tc.variables["ENABLE_WEL"] = "OFF" - tc.variables["ENABLE_PDH"] = "OFF" - tc.variables["ENABLE_SMB"] = "OFF" - elif self.settings.os == "Linux": - tc.variables["ENABLE_SYSTEMD"] = "OFF" - tc.variables["ENABLE_PROCFS"] = "OFF" - - tc.variables["ENABLE_LIBARCHIVE"] = "OFF" - tc.variables["ENABLE_LZMA"] = "OFF" - tc.variables["ENABLE_GPS"] = "OFF" - tc.variables["ENABLE_COAP"] = "OFF" - tc.variables["ENABLE_SQL"] = "OFF" - tc.variables["ENABLE_MQTT"] = "OFF" - tc.variables["ENABLE_PCAP"] = "OFF" - tc.variables["ENABLE_LIBRDKAFKA"] = "OFF" - tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" - tc.variables["ENABLE_PYTHON_SCRIPTING"] = "OFF" - tc.variables["ENABLE_SENSORS"] = "OFF" - tc.variables["ENABLE_USB_CAMERA"] = "OFF" - tc.variables["ENABLE_AWS"] = "OFF" - tc.variables["ENABLE_OPENCV"] = "OFF" - tc.variables["ENABLE_BUSTACHE"] = "OFF" - tc.variables["ENABLE_SFTP"] = "OFF" - tc.variables["ENABLE_AZURE"] = "OFF" - tc.variables["ENABLE_ENCRYPT_CONFIG"] = "OFF" - tc.variables["ENABLE_SPLUNK"] = "OFF" - tc.variables["ENABLE_ELASTICSEARCH"] = "OFF" - tc.variables["ENABLE_GCP"] = "OFF" - tc.variables["ENABLE_KUBERNETES"] = "OFF" - tc.variables["ENABLE_TEST_PROCESSORS"] = "OFF" - tc.variables["ENABLE_PROMETHEUS"] = "OFF" - tc.variables["ENABLE_GRAFANA_LOKI"] = "OFF" - tc.variables["ENABLE_GRPC_FOR_LOKI"] = "OFF" - tc.variables["ENABLE_CONTROLLER"] = "OFF" - + tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" + tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" tc.generate() def build(self): From ba0879e72ad205e686fd133deeb1e154e9ab4faa Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 8 Jun 2024 18:03:35 -0700 Subject: [PATCH 07/26] Able to build minifi w most default ENABLED options except 2 of them After I added new MiNiFi options for MINIFI_LIBCURL_SOURCE, MINIFI_OPENSSL_SOURCE and MINIFI_ZLIB_SOURCE all set to CONAN, I tried building MiNiFi with all the default ENABLED options, but there were two I needed to switch to OFF. I had to switch ENABLE_LIBARCHIVE and ENABLE_AWS to OFF because they failed to build. I do want to note that on my much larger PR where I built majority of MiNiFi using conan to install most of the external libs, I was able to build MiNiFi with openssl, libcurl, zlib and libarchive with no issues. I was also able to keep AWS enabled. I can do a follow PR where I bring back ENABLE_LIBARCHIVE and ENABLE_AWS to where we can set them to ON and be able to successfully build MiNiFi using conan --- cmake/GetOpenSSL.cmake | 3 +-- conanfile.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index ab1d7bb3bd..9c8b2542d2 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -18,8 +18,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") - # include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) - find_package(OpenSSL REQUIRED) + include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") include(BundledOpenSSL) diff --git a/conanfile.py b/conanfile.py index afe42203af..2d9be7dab1 100644 --- a/conanfile.py +++ b/conanfile.py @@ -19,9 +19,14 @@ class MiNiFiCppMain(ConanFile): def generate(self): tc = CMakeToolchain(self) + tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + + tc.variables["ENABLE_LIBARCHIVE"] = "OFF" + tc.variables["ENABLE_AWS"] = "OFF" + tc.generate() def build(self): From 6a093086737d6ddf9cea9f20af7ba78cc5c0ad86 Mon Sep 17 00:00:00 2001 From: james94 Date: Tue, 25 Jun 2024 10:25:11 -0700 Subject: [PATCH 08/26] Rebased on main, verified can build MiNiFi w conan & cmake Followed Marton's suggested steps to remove my merge commit, then do an interactive rebase to only include my commits and remove Gabor's commits to keep a clean PR-1793 for my MINIFICPP-2346-P1 branch. For building MiNiFi using conan with conan's cmake wrapper, I disabled building TESTS and disabled expression language since they were causing compile issues. I believe I resolved them in my follow up PR-1813, which will only be ready for further review once my PR-1793 is ready for merge and merged. After disabling these two components of the build, I was able to build MiNiFi successfully. I verified I could still build MiNiFi using standalone cmake and also verified I could still build it with TESTS enabled and expression language enabled and the other necessary features that needed to be enabled to build MiNIFi with TESTS enabled. For the MiNiFi standalone cmake build, I disabled features that werent needed like extra extensions for building faster. --- conanfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conanfile.py b/conanfile.py index 2d9be7dab1..e802c625ec 100644 --- a/conanfile.py +++ b/conanfile.py @@ -24,8 +24,10 @@ def generate(self): tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["SKIP_TESTS"] = "ON" tc.variables["ENABLE_LIBARCHIVE"] = "OFF" tc.variables["ENABLE_AWS"] = "OFF" + tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "OFF" tc.generate() From 0ea3ab9d737a37878250d846f3d590eebaa8df9a Mon Sep 17 00:00:00 2001 From: James Guzman Date: Wed, 26 Jun 2024 13:27:13 -0700 Subject: [PATCH 09/26] Update cmake/MiNiFiOptions.cmake's add_minifi_multi_option w Comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update cmake/MiNiFiOptions.cmake for add_multi_option with Comments portraying the Meaning of Each Option. Co-authored-by: Márton Szász --- cmake/MiNiFiOptions.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 91fdb411cb..ab03827fbd 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -132,6 +132,9 @@ set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) +# BUILD: Fetch and build from source using CMake FetchContent or ExternalProject +# SYSTEM: Use find_package to use the system version +# CONAN: Use Conan packages add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") From 6ed214d460fc6d60363882e4359acbf5455bba17 Mon Sep 17 00:00:00 2001 From: james94 Date: Fri, 9 Aug 2024 13:47:40 -0700 Subject: [PATCH 10/26] Updated MiNiFi's conanfile.py & CMake to create conan package --- CMakeLists.txt | 20 +++-- cmake/BuildTests.cmake | 7 +- cmake/Bustache.cmake | 4 +- cmake/GetCatch2.cmake | 28 ++++++ cmake/GetCivetWeb.cmake | 26 ++++++ cmake/GetFmt.cmake | 26 ++++++ cmake/GetLibXml2.cmake | 27 ++++++ cmake/GetSpdlog.cmake | 29 ++++++ cmake/MiNiFiOptions.cmake | 7 +- cmake/Spdlog.cmake | 2 +- conanfile.py | 89 +++++++++++++++++-- controller/tests/CMakeLists.txt | 2 +- encrypt-config/tests/CMakeLists.txt | 2 +- extensions/aws/tests/CMakeLists.txt | 2 +- extensions/azure/tests/CMakeLists.txt | 2 +- extensions/bustache/tests/CMakeLists.txt | 2 +- extensions/civetweb/CMakeLists.txt | 2 +- extensions/civetweb/tests/CMakeLists.txt | 2 +- extensions/elasticsearch/tests/CMakeLists.txt | 2 +- .../execute-process/test/CMakeLists.txt | 2 +- .../expression-language/tests/CMakeLists.txt | 4 +- extensions/grafana-loki/tests/CMakeLists.txt | 2 +- extensions/kubernetes/tests/CMakeLists.txt | 2 +- extensions/libarchive/tests/CMakeLists.txt | 2 +- extensions/librdkafka/tests/CMakeLists.txt | 2 +- extensions/lua/tests/CMakeLists.txt | 2 +- extensions/mqtt/tests/CMakeLists.txt | 2 +- extensions/opc/CMakeLists.txt | 2 +- extensions/opencv/tests/CMakeLists.txt | 2 +- extensions/pdh/tests/CMakeLists.txt | 2 +- extensions/procfs/tests/CMakeLists.txt | 2 +- extensions/prometheus/tests/CMakeLists.txt | 2 +- extensions/python/tests/CMakeLists.txt | 4 +- extensions/rocksdb-repos/tests/CMakeLists.txt | 2 +- extensions/script/tests/CMakeLists.txt | 2 +- extensions/sftp/tests/CMakeLists.txt | 2 +- extensions/smb/tests/CMakeLists.txt | 2 +- extensions/splunk/tests/CMakeLists.txt | 2 +- extensions/sql/tests/CMakeLists.txt | 2 +- .../standard-processors/tests/CMakeLists.txt | 4 +- extensions/systemd/tests/CMakeLists.txt | 2 +- .../windows-event-log/tests/CMakeLists.txt | 2 +- libminifi/CMakeLists.txt | 2 +- libminifi/test/flow-tests/CMakeLists.txt | 2 +- libminifi/test/integration/CMakeLists.txt | 2 +- libminifi/test/libtest/unit/CMakeLists.txt | 2 +- .../test/persistence-tests/CMakeLists.txt | 2 +- libminifi/test/schema-tests/CMakeLists.txt | 2 +- libminifi/test/unit/CMakeLists.txt | 2 +- minifi_main/tests/CMakeLists.txt | 2 +- 50 files changed, 288 insertions(+), 61 deletions(-) create mode 100644 cmake/GetCatch2.cmake create mode 100644 cmake/GetCivetWeb.cmake create mode 100644 cmake/GetFmt.cmake create mode 100644 cmake/GetLibXml2.cmake create mode 100644 cmake/GetSpdlog.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e3f5e7c63a..ac1e5e522b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -261,7 +261,8 @@ get_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy") # spdlog -include(Spdlog) +include(GetSpdlog) +get_spdlog() # yaml-cpp include(BundledYamlCpp) @@ -340,14 +341,15 @@ include(Extensions) add_subdirectory(libminifi) -if (ENABLE_ALL OR ENABLE_AZURE) - include(BundledLibXml2) - use_bundled_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +if ((ENABLE_OPENWSMAN AND ENABLE_CIVET) OR ENABLE_ALL OR ENABLE_AZURE) + include(GetLibXml2) + get_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy") endif() if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_CIVET) - include(CivetWeb) + include(GetCivetWeb) + get_civetweb() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy") endif() @@ -594,9 +596,11 @@ set(CPACK_PACKAGE_VENDOR "Apache NiFi") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) -set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) -set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") +if(NOT MINIFI_BUILD_CONAN_PACKAGE) + set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) + set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") +endif() if(NOT WIN32) set(CPACK_PACKAGE_FILE_NAME "${ASSEMBLY_BASE_NAME}") set(CPACK_GENERATOR "TGZ") diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index e55500242a..c18079c4e5 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -include(Catch2) +include(GetCatch2) +get_catch2() ### test functions MACRO(GETSOURCEFILES result curdir) @@ -76,7 +77,7 @@ function(createTests testName) target_link_libraries(${testName} ${CMAKE_DL_LIBS}) target_wholearchive_library(${testName} libminifi-unittest) - target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads Catch2::Catch2WithMain) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") @@ -88,7 +89,7 @@ function(createIntegrationTests testName) target_link_libraries(${testName} ${CMAKE_DL_LIBS}) target_wholearchive_library(${testName} libminifi-integrationtest) - target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads Catch2::Catch2WithMain) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") diff --git a/cmake/Bustache.cmake b/cmake/Bustache.cmake index 1160287752..6c037af6c9 100644 --- a/cmake/Bustache.cmake +++ b/cmake/Bustache.cmake @@ -17,7 +17,9 @@ # under the License. # include(FetchContent) -include(fmt) +include(GetFmt) +get_fmt() + set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) FetchContent_Declare(Bustache GIT_REPOSITORY https://github.com/jamboree/bustache.git diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake new file mode 100644 index 0000000000..e0745ad246 --- /dev/null +++ b/cmake/GetCatch2.cmake @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_catch2) + if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Catch2 external lib") + include(${CMAKE_BINARY_DIR}/Catch2Config.cmake) + message("Catch2_INCLUDE_DIRS = ${Catch2_INCLUDE_DIRS}") + set(CATCH2_INCLUDE_DIRS "${Catch2_INCLUDE_DIRS}" CACHE STRING "" FORCE) + elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib") + include(Catch2) + endif() +endfunction(get_catch2) diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake new file mode 100644 index 0000000000..419c5d834c --- /dev/null +++ b/cmake/GetCivetWeb.cmake @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_civetweb) + if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") + include(${CMAKE_BINARY_DIR}/civetweb-config.cmake) + elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib") + include(CivetWeb) + endif() +endfunction(get_civetweb) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake new file mode 100644 index 0000000000..bb9f7da346 --- /dev/null +++ b/cmake/GetFmt.cmake @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_fmt) + if(MINIFI_FMT_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Fmt external lib") + include(${CMAKE_BINARY_DIR}/fmt-config.cmake) + elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib") + include(fmt) + endif() +endfunction(get_fmt) diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake new file mode 100644 index 0000000000..6381e6dccd --- /dev/null +++ b/cmake/GetLibXml2.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_libxml2 SOURCE_DIR BINARY_DIR) + if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") + include(${CMAKE_BINARY_DIR}/libxml2-config.cmake) + elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib") + include(BundledLibXml2) + use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_libxml2) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake new file mode 100644 index 0000000000..a30d1215de --- /dev/null +++ b/cmake/GetSpdlog.cmake @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_spdlog) + include(GetFmt) + get_fmt() + + if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Spdlog external lib") + include(${CMAKE_BINARY_DIR}/spdlog-config.cmake) + elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") + include(Spdlog) + endif() +endfunction(get_spdlog) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index ab03827fbd..345dbcc047 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -135,10 +135,15 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values # BUILD: Fetch and build from source using CMake FetchContent or ExternalProject # SYSTEM: Use find_package to use the system version # CONAN: Use Conan packages +add_minifi_option(MINIFI_BUILD_CONAN_PACKAGE "Instructs MiNiFi to install external libs, build & create MiNiFi conan package" OFF) add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") - +add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_CATCH2_SOURCE "Retrieves Catch2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_SPDLOG_SOURCE "Retrieves Spdlog from provided source" "BUILD;SYSTEM;CONAN" "BUILD") # Docker options diff --git a/cmake/Spdlog.cmake b/cmake/Spdlog.cmake index 07eb0f103a..3b07298297 100644 --- a/cmake/Spdlog.cmake +++ b/cmake/Spdlog.cmake @@ -17,7 +17,7 @@ # under the License. # include(FetchContent) -include(fmt) + set(SPDLOG_FMT_EXTERNAL ON CACHE STRING "" FORCE) FetchContent_Declare(Spdlog URL https://github.com/gabime/spdlog/archive/refs/tags/v1.12.0.tar.gz diff --git a/conanfile.py b/conanfile.py index e802c625ec..2aa480d114 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,14 +1,27 @@ from conan import ConanFile from conan.tools.env import VirtualRunEnv from conan.tools.cmake import CMake, CMakeToolchain +from conan.tools.files import collect_libs, copy +import os required_conan_version = ">=2.1.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0") +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", + "catch2/3.5.4", "fmt/10.2.1", "spdlog/1.14.0") + +shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", + "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", + "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", + "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", + "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", + "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", + "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", + "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", + "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") class MiNiFiCppMain(ConanFile): - name = "minifi-cpp-main" - version = "0.99.0" + name = "minifi-cpp" + version = "0.99.1" license = "Apache-2.0" requires = shared_requires settings = "os", "compiler", "build_type", "arch" @@ -17,17 +30,67 @@ class MiNiFiCppMain(ConanFile): default_options = {"shared": False, "fPIC": True,} + exports_sources = shared_sources + def generate(self): tc = CMakeToolchain(self) + tc.variables["MINIFI_BUILD_CONAN_PACKAGE"] = "ON" tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" + tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" + tc.variables["MINIFI_CATCH2_SOURCE"] = "CONAN" + tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" + tc.variables["MINIFI_SPDLOG_SOURCE"] = "CONAN" + + tc.variables["SKIP_TESTS"] = "OFF" + tc.variables["ENABLE_OPENWSMAN"] = "ON" + tc.variables["ENABLE_CIVET"] = "ON" + tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" + tc.variables["ENABLE_BZIP2"] = "ON" + tc.variables["ENABLE_ROCKSDB"] = "ON" + tc.variables["BUILD_ROCKSDB"] = "ON" - tc.variables["SKIP_TESTS"] = "ON" tc.variables["ENABLE_LIBARCHIVE"] = "OFF" tc.variables["ENABLE_AWS"] = "OFF" - tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "OFF" + + tc.variables["ENABLE_OPC"] = "OFF" + + if self.settings.os == "Windows": + tc.variables["ENABLE_WEL"] = "OFF" + tc.variables["ENABLE_PDH"] = "OFF" + tc.variables["ENABLE_SMB"] = "OFF" + elif self.settings.os == "Linux": + tc.variables["ENABLE_SYSTEMD"] = "OFF" + tc.variables["ENABLE_PROCFS"] = "OFF" + + tc.variables["ENABLE_LZMA"] = "OFF" + tc.variables["ENABLE_GPS"] = "OFF" + tc.variables["ENABLE_COAP"] = "OFF" + tc.variables["ENABLE_SQL"] = "OFF" + tc.variables["ENABLE_MQTT"] = "OFF" + tc.variables["ENABLE_PCAP"] = "OFF" + tc.variables["ENABLE_LIBRDKAFKA"] = "OFF" + tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" + tc.variables["ENABLE_PYTHON_SCRIPTING"] = "OFF" + tc.variables["ENABLE_SENSORS"] = "OFF" + tc.variables["ENABLE_USB_CAMERA"] = "OFF" + tc.variables["ENABLE_OPENCV"] = "OFF" + tc.variables["ENABLE_BUSTACHE"] = "OFF" + tc.variables["ENABLE_SFTP"] = "OFF" + tc.variables["ENABLE_AZURE"] = "OFF" + tc.variables["ENABLE_ENCRYPT_CONFIG"] = "OFF" + tc.variables["ENABLE_SPLUNK"] = "OFF" + tc.variables["ENABLE_ELASTICSEARCH"] = "OFF" + tc.variables["ENABLE_GCP"] = "OFF" + tc.variables["ENABLE_KUBERNETES"] = "OFF" + tc.variables["ENABLE_TEST_PROCESSORS"] = "OFF" + tc.variables["ENABLE_PROMETHEUS"] = "OFF" + tc.variables["ENABLE_GRAFANA_LOKI"] = "OFF" + tc.variables["ENABLE_GRPC_FOR_LOKI"] = "OFF" + tc.variables["ENABLE_CONTROLLER"] = "OFF" tc.generate() @@ -35,3 +98,19 @@ def build(self): cmake = CMake(self) cmake.configure() cmake.build() + + def package(self): + cmake = CMake(self) + include_dir = os.path.join(self.source_folder) + built_dir = os.path.join(self.source_folder, self.folders.build) + copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + cmake.install() + + def package_info(self): + self.cpp_info.libs = collect_libs(self, folder=os.path.join(self.package_folder, "lib")) + self.cpp_info.set_property("cmake_file_name", "minifi-cpp") + self.cpp_info.set_property("cmake_target_name", "minifi-cpp::minifi-cpp") + self.cpp_info.set_property("pkg_config_name", "minifi-cpp") diff --git a/controller/tests/CMakeLists.txt b/controller/tests/CMakeLists.txt index c5fca51a6f..056969fa60 100644 --- a/controller/tests/CMakeLists.txt +++ b/controller/tests/CMakeLists.txt @@ -36,7 +36,7 @@ foreach(testfile ${CONTROLLER_TESTS}) createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} libminifi-unittest) + target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR CONTROLLER_TEST_COUNT "${CONTROLLER_TEST_COUNT}+1") diff --git a/encrypt-config/tests/CMakeLists.txt b/encrypt-config/tests/CMakeLists.txt index 175e70cff2..91d8e39dd3 100644 --- a/encrypt-config/tests/CMakeLists.txt +++ b/encrypt-config/tests/CMakeLists.txt @@ -31,7 +31,7 @@ foreach(testfile ${ENCRYPT_CONFIG_TESTS}) createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} libminifi-unittest) + target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR ENCRYPT_CONFIG_TEST_COUNT "${ENCRYPT_CONFIG_TEST_COUNT}+1") diff --git a/extensions/aws/tests/CMakeLists.txt b/extensions/aws/tests/CMakeLists.txt index d26c0ebee9..7bc116b616 100644 --- a/extensions/aws/tests/CMakeLists.txt +++ b/extensions/aws/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${AWS_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-aws) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/azure/tests/CMakeLists.txt b/extensions/azure/tests/CMakeLists.txt index 730cf34074..9bce4a99d4 100644 --- a/extensions/azure/tests/CMakeLists.txt +++ b/extensions/azure/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${AZURE_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") target_compile_features(${testfilename} PRIVATE cxx_std_14) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-azure) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/bustache/tests/CMakeLists.txt b/extensions/bustache/tests/CMakeLists.txt index 4f493df658..5ec7664f38 100644 --- a/extensions/bustache/tests/CMakeLists.txt +++ b/extensions/bustache/tests/CMakeLists.txt @@ -30,7 +30,7 @@ FOREACH(testfile ${BUSTACHE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR BUSTACHE-EXTENSIONS_TEST_COUNT "${BUSTACHE-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/civetweb/CMakeLists.txt b/extensions/civetweb/CMakeLists.txt index 0eab2b98cd..c6f45d280c 100644 --- a/extensions/civetweb/CMakeLists.txt +++ b/extensions/civetweb/CMakeLists.txt @@ -32,7 +32,7 @@ target_include_directories(minifi-civet-extensions BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/thirdparty/ ${civetweb_SOURCE_DIR}/include ./include) -target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp civetweb::c-library) +target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp) register_extension(minifi-civet-extensions CIVETWEB CIVETWEB "This enables ListenHTTP" "extensions/civetweb/tests") register_extension_linter(minifi-civet-extensions-linter) diff --git a/extensions/civetweb/tests/CMakeLists.txt b/extensions/civetweb/tests/CMakeLists.txt index 95f7af7d87..23dceab36a 100644 --- a/extensions/civetweb/tests/CMakeLists.txt +++ b/extensions/civetweb/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR CIVETWEB-EXTENSIONS_TEST_COUNT "${CIVETWEB-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) # Copy test resources diff --git a/extensions/elasticsearch/tests/CMakeLists.txt b/extensions/elasticsearch/tests/CMakeLists.txt index 58a2120131..82a7ff6432 100644 --- a/extensions/elasticsearch/tests/CMakeLists.txt +++ b/extensions/elasticsearch/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${ELASTICSEARCH_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-elasticsearch) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/execute-process/test/CMakeLists.txt b/extensions/execute-process/test/CMakeLists.txt index 05d5f93480..0467357804 100644 --- a/extensions/execute-process/test/CMakeLists.txt +++ b/extensions/execute-process/test/CMakeLists.txt @@ -22,7 +22,7 @@ set_target_properties(EchoParameters PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAK add_minifi_executable(ExecuteProcessTests ExecuteProcessTests.cpp) -target_link_libraries(ExecuteProcessTests minifi-execute-process minifi-standard-processors Catch2WithMain) +target_link_libraries(ExecuteProcessTests minifi-execute-process minifi-standard-processors Catch2::Catch2WithMain) target_include_directories(ExecuteProcessTests PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test") add_test(NAME ExecuteProcessTests COMMAND ExecuteProcessTests WORKING_DIRECTORY ${TEST_DIR}) createTests(ExecuteProcessTests) diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt index 2f96ee3df2..c5e08eca89 100644 --- a/extensions/expression-language/tests/CMakeLists.txt +++ b/extensions/expression-language/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") @@ -52,7 +52,7 @@ FOREACH(testfile ${INT_EXPRESSION_LANGUAGE_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createIntegrationTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") diff --git a/extensions/grafana-loki/tests/CMakeLists.txt b/extensions/grafana-loki/tests/CMakeLists.txt index 8aa24fd887..d32fea604f 100644 --- a/extensions/grafana-loki/tests/CMakeLists.txt +++ b/extensions/grafana-loki/tests/CMakeLists.txt @@ -34,7 +34,7 @@ FOREACH(testfile ${GRAFANA_LOKI_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-grafana-loki) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/kubernetes/tests/CMakeLists.txt b/extensions/kubernetes/tests/CMakeLists.txt index d5dd9abcab..0c2c0d295e 100644 --- a/extensions/kubernetes/tests/CMakeLists.txt +++ b/extensions/kubernetes/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(TEST_FILE ${KUBERNETES_UNIT_TESTS}) target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/kubernetes") target_link_libraries(${TEST_TARGET} minifi-kubernetes-extensions) - target_link_libraries(${TEST_TARGET} Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR KUBERNETES_UNIT_TEST_COUNT "${KUBERNETES_UNIT_TEST_COUNT}+1") diff --git a/extensions/libarchive/tests/CMakeLists.txt b/extensions/libarchive/tests/CMakeLists.txt index 5c1ca342fe..1358371492 100644 --- a/extensions/libarchive/tests/CMakeLists.txt +++ b/extensions/libarchive/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${ARCHIVE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-archive-extensions) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/librdkafka/tests/CMakeLists.txt b/extensions/librdkafka/tests/CMakeLists.txt index 7f8053884c..c8d0798951 100644 --- a/extensions/librdkafka/tests/CMakeLists.txt +++ b/extensions/librdkafka/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${KAFKA_INTEGRATION_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/librdkafka") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-rdkafka-extensions) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/lua/tests/CMakeLists.txt b/extensions/lua/tests/CMakeLists.txt index 38d6979906..9922d94e35 100644 --- a/extensions/lua/tests/CMakeLists.txt +++ b/extensions/lua/tests/CMakeLists.txt @@ -27,7 +27,7 @@ FOREACH(testfile ${EXECUTESCRIPT_LUA_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/lua") target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${SOL2_INCLUDE_DIR}") target_link_libraries(${testfilename} minifi-lua-script-extension minifi-script-extension) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/mqtt/tests/CMakeLists.txt b/extensions/mqtt/tests/CMakeLists.txt index d5b408a64a..913eb1ef77 100644 --- a/extensions/mqtt/tests/CMakeLists.txt +++ b/extensions/mqtt/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${MQTT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../../libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-mqtt-extensions) target_link_libraries(${testfilename} minifi-standard-processors) add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/opc/CMakeLists.txt b/extensions/opc/CMakeLists.txt index 9f1267cf92..0d120de83d 100644 --- a/extensions/opc/CMakeLists.txt +++ b/extensions/opc/CMakeLists.txt @@ -38,7 +38,7 @@ file(GLOB SOURCES "src/*.cpp") add_minifi_library(minifi-opc-extensions SHARED ${SOURCES}) target_link_libraries(minifi-opc-extensions ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog open62541::open62541) +target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog::spdlog open62541::open62541) register_extension(minifi-opc-extensions "OPC EXTENSIONS" OPC-EXTENSIONS "This enables OPC-UA support") register_extension_linter(minifi-opc-extensions-linter) diff --git a/extensions/opencv/tests/CMakeLists.txt b/extensions/opencv/tests/CMakeLists.txt index d095c7e9a2..cc34d8550f 100644 --- a/extensions/opencv/tests/CMakeLists.txt +++ b/extensions/opencv/tests/CMakeLists.txt @@ -31,6 +31,6 @@ FOREACH(testfile ${OPENCV_TESTS}) createTests("${testfilename}") MATH(EXPR OPENCV_TEST_COUNT "${OPENCV_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) ENDFOREACH() message("-- Finished building ${OPENCV_TEST_COUNT} OpenCV related test file(s)...") diff --git a/extensions/pdh/tests/CMakeLists.txt b/extensions/pdh/tests/CMakeLists.txt index 63686553fa..5ba1b57585 100644 --- a/extensions/pdh/tests/CMakeLists.txt +++ b/extensions/pdh/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${PDH_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/pdh") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-pdh) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PDH_TEST_COUNT "${PDH_TEST_COUNT}+1") diff --git a/extensions/procfs/tests/CMakeLists.txt b/extensions/procfs/tests/CMakeLists.txt index ad51a869d3..060f51dadd 100644 --- a/extensions/procfs/tests/CMakeLists.txt +++ b/extensions/procfs/tests/CMakeLists.txt @@ -27,7 +27,7 @@ FOREACH(testfile ${PROCFS_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/procfs") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-procfs) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PROCFS_TEST_COUNT "${PROCFS_TEST_COUNT}+1") diff --git a/extensions/prometheus/tests/CMakeLists.txt b/extensions/prometheus/tests/CMakeLists.txt index 29b3185c43..ebe0db595d 100644 --- a/extensions/prometheus/tests/CMakeLists.txt +++ b/extensions/prometheus/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${PROMETHEUS_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/prometheus") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-prometheus) MATH(EXPR PROMETHEUS_TEST_COUNT "${PROMETHEUS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") diff --git a/extensions/python/tests/CMakeLists.txt b/extensions/python/tests/CMakeLists.txt index 06cc89e44a..3c5082aacd 100644 --- a/extensions/python/tests/CMakeLists.txt +++ b/extensions/python/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${EXECUTESCRIPT_PYTHON_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2::Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -44,7 +44,7 @@ FOREACH(testfile ${EXECUTEPYTHONPROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2::Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/rocksdb-repos/tests/CMakeLists.txt b/extensions/rocksdb-repos/tests/CMakeLists.txt index 53b47dbbcd..18088e0c91 100644 --- a/extensions/rocksdb-repos/tests/CMakeLists.txt +++ b/extensions/rocksdb-repos/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${ROCKSDB_UNIT_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/rocksdb-repos/") target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${ROCKSDB_THIRDPARTY_ROOT}/include") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-rocksdb-repos) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR ROCKSDB_TEST_COUNT "${ROCKSDB_TEST_COUNT}+1") diff --git a/extensions/script/tests/CMakeLists.txt b/extensions/script/tests/CMakeLists.txt index 2157777555..5c88322264 100644 --- a/extensions/script/tests/CMakeLists.txt +++ b/extensions/script/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${EXECUTE_SCRIPT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-script-extension) MATH(EXPR EXECUTE_SCRIPT_TEST_COUNT "${EXECUTE_SCRIPT_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/sftp/tests/CMakeLists.txt b/extensions/sftp/tests/CMakeLists.txt index a8767d7bcb..7276b275b5 100644 --- a/extensions/sftp/tests/CMakeLists.txt +++ b/extensions/sftp/tests/CMakeLists.txt @@ -40,7 +40,7 @@ if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE target_include_directories(${testfilename} BEFORE PRIVATE ./tools) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain core-minifi sftp-test-tools) + target_link_libraries(${testfilename} Catch2::Catch2WithMain core-minifi sftp-test-tools) target_link_libraries(${testfilename} minifi-sftp) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/smb/tests/CMakeLists.txt b/extensions/smb/tests/CMakeLists.txt index d1d02b6953..1a5ab5eec5 100644 --- a/extensions/smb/tests/CMakeLists.txt +++ b/extensions/smb/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${SMB_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/smb") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-smb) target_link_libraries(${testfilename} minifi-standard-processors Netapi32) MATH(EXPR SMB_TEST_COUNT "${SMB_TEST_COUNT}+1") diff --git a/extensions/splunk/tests/CMakeLists.txt b/extensions/splunk/tests/CMakeLists.txt index f00d2740b4..313844c52d 100644 --- a/extensions/splunk/tests/CMakeLists.txt +++ b/extensions/splunk/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${SPLUNK_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-splunk) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/sql/tests/CMakeLists.txt b/extensions/sql/tests/CMakeLists.txt index 81184f0664..2996cbad1e 100644 --- a/extensions/sql/tests/CMakeLists.txt +++ b/extensions/sql/tests/CMakeLists.txt @@ -34,7 +34,7 @@ foreach(testfile ${SQL_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-sql-mocks) target_link_libraries(${testfilename} minifi-sql) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/standard-processors/tests/CMakeLists.txt b/extensions/standard-processors/tests/CMakeLists.txt index 4eff161456..a17229003f 100644 --- a/extensions/standard-processors/tests/CMakeLists.txt +++ b/extensions/standard-processors/tests/CMakeLists.txt @@ -36,7 +36,7 @@ FOREACH(testfile ${PROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE JOLT_TESTS_DIR="${jolt_tests_SOURCE_DIR}/jolt-core/src/test/resources/json/shiftr") @@ -66,7 +66,7 @@ FOREACH(testfile ${PROCESSOR_INTEGRATION_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE ./include) createIntegrationTests("${testfilename}") target_link_libraries(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") diff --git a/extensions/systemd/tests/CMakeLists.txt b/extensions/systemd/tests/CMakeLists.txt index 9d245ea236..77160a591b 100644 --- a/extensions/systemd/tests/CMakeLists.txt +++ b/extensions/systemd/tests/CMakeLists.txt @@ -26,7 +26,7 @@ function(add_systemd_test testfile) target_link_libraries(${TEST_TARGET} minifi-standard-processors) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") - target_link_libraries(${TEST_TARGET} Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) endfunction() add_systemd_test("ConsumeJournaldTest.cpp") diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 07c40712f5..1939cd3553 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -36,7 +36,7 @@ FOREACH(testfile ${WEL_TESTS}) createTests("${testfilename}") target_link_libraries(${testfilename} minifi-wel) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries (${testfilename} ${LIBMINIFI} Catch2WithMain) + target_link_libraries (${testfilename} ${LIBMINIFI} Catch2::Catch2WithMain) MATH(EXPR WEL_TEST_COUNT "${WEL_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 3a7778236d..73a3fc2628 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -76,7 +76,7 @@ endif() include(RangeV3) include(Asio) include(MagicEnum) -list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON) +list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog::spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON fmt::fmt) if(NOT WIN32) list(APPEND LIBMINIFI_LIBRARIES OSSP::libuuid++) endif() diff --git a/libminifi/test/flow-tests/CMakeLists.txt b/libminifi/test/flow-tests/CMakeLists.txt index 67845af565..9111ffdefc 100644 --- a/libminifi/test/flow-tests/CMakeLists.txt +++ b/libminifi/test/flow-tests/CMakeLists.txt @@ -31,7 +31,7 @@ FOREACH(testfile ${FLOW_TESTS}) target_link_libraries(${testfilename} minifi-rocksdb-repos) endif() - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR FLOW_TEST_COUNT "${FLOW_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") ENDFOREACH() diff --git a/libminifi/test/integration/CMakeLists.txt b/libminifi/test/integration/CMakeLists.txt index bd7bf697ad..c52f40cf9e 100644 --- a/libminifi/test/integration/CMakeLists.txt +++ b/libminifi/test/integration/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${INTEGRATION_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} Catch2WithMain Threads::Threads) + target_link_libraries(${testfilename} Catch2::Catch2WithMain Threads::Threads) target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") if(NOT ${testfilename} IN_LIST TESTS_WITH_LIBARCHIVE_DEPENDENCY) add_test(NAME "${testfilename}" COMMAND "${testfilename}") diff --git a/libminifi/test/libtest/unit/CMakeLists.txt b/libminifi/test/libtest/unit/CMakeLists.txt index 75847685e3..dccb78a9ac 100644 --- a/libminifi/test/libtest/unit/CMakeLists.txt +++ b/libminifi/test/libtest/unit/CMakeLists.txt @@ -18,7 +18,7 @@ # file(GLOB TEST_BASE_SOURCES "*.cpp") add_minifi_library(libminifi-unittest STATIC ${TEST_BASE_SOURCES}) -target_link_libraries(libminifi-unittest core-minifi Catch2) +target_link_libraries(libminifi-unittest core-minifi Catch2::Catch2WithMain) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") if(WIN32) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") diff --git a/libminifi/test/persistence-tests/CMakeLists.txt b/libminifi/test/persistence-tests/CMakeLists.txt index c2913bd6d6..c2b51a2afe 100644 --- a/libminifi/test/persistence-tests/CMakeLists.txt +++ b/libminifi/test/persistence-tests/CMakeLists.txt @@ -32,7 +32,7 @@ FOREACH(testfile ${PERSISTENCE_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-rocksdb-repos) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR PERSISTENCE_TEST_COUNT "${PERSISTENCE_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/libminifi/test/schema-tests/CMakeLists.txt b/libminifi/test/schema-tests/CMakeLists.txt index a5befa0829..0a53ed1f10 100644 --- a/libminifi/test/schema-tests/CMakeLists.txt +++ b/libminifi/test/schema-tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${SCHEMA_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} nlohmann_json_schema_validator) MATH(EXPR SCHEMA_TEST_COUNT "${SCHEMA_TEST_COUNT}+1") diff --git a/libminifi/test/unit/CMakeLists.txt b/libminifi/test/unit/CMakeLists.txt index 08e93d4cab..94308c0a49 100644 --- a/libminifi/test/unit/CMakeLists.txt +++ b/libminifi/test/unit/CMakeLists.txt @@ -30,7 +30,7 @@ FOREACH(testfile ${UNIT_TESTS}) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") ENDFOREACH() diff --git a/minifi_main/tests/CMakeLists.txt b/minifi_main/tests/CMakeLists.txt index 7620162ec5..12afbf7b72 100644 --- a/minifi_main/tests/CMakeLists.txt +++ b/minifi_main/tests/CMakeLists.txt @@ -23,7 +23,7 @@ FOREACH(TEST_FILE ${MINIFI_MAIN_UNIT_TESTS}) get_filename_component(TEST_TARGET "${TEST_FILE}" NAME_WE) add_minifi_executable("${TEST_TARGET}" "${TEST_FILE}" "../TableFormatter.cpp") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/minifi_main") - target_link_libraries(${TEST_TARGET} core-minifi Catch2WithMain) + target_link_libraries(${TEST_TARGET} core-minifi Catch2::Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR MINIFI_MAIN_UNIT_TEST_COUNT "${MINIFI_MAIN_UNIT_TEST_COUNT}+1") From 0cb9c54d1bf9319aaee72cb826563a2a38ecb15c Mon Sep 17 00:00:00 2001 From: james94 Date: Fri, 9 Aug 2024 14:47:46 -0700 Subject: [PATCH 11/26] Updated conan version support to be at least 2.0 Thus, other people using Nix to install conan can now install conan 2.0.17 or alternative supported version. I was able to build MiNiFi c++ using conan 2.0.17. In process of testing creating the conan package too --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 2aa480d114..21addfc215 100644 --- a/conanfile.py +++ b/conanfile.py @@ -4,7 +4,7 @@ from conan.tools.files import collect_libs, copy import os -required_conan_version = ">=2.1.0" +required_conan_version = ">=2.0" shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "catch2/3.5.4", "fmt/10.2.1", "spdlog/1.14.0") From dea85ba99e637c06a5171aaa6cae4ec6024c9956 Mon Sep 17 00:00:00 2001 From: james94 Date: Fri, 9 Aug 2024 17:56:34 -0700 Subject: [PATCH 12/26] Verified Can Create MiNiFi Conan Package after removing MINIFI_BUILD_CONAN_PACKAGE One of the concerns was that we wanted to make sure that README.md, LICENSE , NOTICE and even the binary ones were included in the MiNiFi C++ conan package. I removed the MINIFI_BUILD_CONAN_PACKAGE condition, so CPACK_PACKAGE_DESCRIPTION_FILE and CPACK_RESOURCE_FILE_LICENSE are not ignored when creating the MiNiFi conan package. This is for README.md and LICENSE.txt and NOTICE files including the binary ones. As an alternative to using CPACK with CMake to make sure these files are included in the MiNiFi conan package, through conanfile.py we assign our shared_sources variable to export_sources and in our py file's package() method, we run cmake.install() to ensure that the source files and binary files assets are copied into the MiNiFi conan package. Also for the MiNiFi conan package, we copy over the .h, .i, .a and .so files into their appropriate include/ folder and lib/ folder. --- CMakeLists.txt | 8 +++----- cmake/MiNiFiOptions.cmake | 1 - conanfile.py | 1 - 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac1e5e522b..4626f4c28f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -596,11 +596,9 @@ set(CPACK_PACKAGE_VENDOR "Apache NiFi") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) -if(NOT MINIFI_BUILD_CONAN_PACKAGE) - set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) - set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") -endif() +set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") if(NOT WIN32) set(CPACK_PACKAGE_FILE_NAME "${ASSEMBLY_BASE_NAME}") set(CPACK_GENERATOR "TGZ") diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 345dbcc047..371f3d507c 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -135,7 +135,6 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values # BUILD: Fetch and build from source using CMake FetchContent or ExternalProject # SYSTEM: Use find_package to use the system version # CONAN: Use Conan packages -add_minifi_option(MINIFI_BUILD_CONAN_PACKAGE "Instructs MiNiFi to install external libs, build & create MiNiFi conan package" OFF) add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") diff --git a/conanfile.py b/conanfile.py index 21addfc215..96f4c929fc 100644 --- a/conanfile.py +++ b/conanfile.py @@ -35,7 +35,6 @@ class MiNiFiCppMain(ConanFile): def generate(self): tc = CMakeToolchain(self) - tc.variables["MINIFI_BUILD_CONAN_PACKAGE"] = "ON" tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" From 125dfd2d49bdcc8a0545efc8d76eba6845488f3b Mon Sep 17 00:00:00 2001 From: james94 Date: Fri, 9 Aug 2024 18:25:41 -0700 Subject: [PATCH 13/26] Refactored conan cmake include packages to find_package() Verified after switching cmake include(...) conan package libs to find_package(...) of that conan package lib name that I could run conan build and conan create successfully --- cmake/GetCatch2.cmake | 2 +- cmake/GetCivetWeb.cmake | 2 +- cmake/GetFmt.cmake | 2 +- cmake/GetLibCURL.cmake | 2 +- cmake/GetLibXml2.cmake | 2 +- cmake/GetOpenSSL.cmake | 2 +- cmake/GetSpdlog.cmake | 2 +- cmake/GetZLIB.cmake | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index e0745ad246..d30c5c8ac4 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -18,7 +18,7 @@ function(get_catch2) if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt Catch2 external lib") - include(${CMAKE_BINARY_DIR}/Catch2Config.cmake) + find_package(Catch2 REQUIRED) message("Catch2_INCLUDE_DIRS = ${Catch2_INCLUDE_DIRS}") set(CATCH2_INCLUDE_DIRS "${Catch2_INCLUDE_DIRS}" CACHE STRING "" FORCE) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake index 419c5d834c..3e03b4fbc5 100644 --- a/cmake/GetCivetWeb.cmake +++ b/cmake/GetCivetWeb.cmake @@ -18,7 +18,7 @@ function(get_civetweb) if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") - include(${CMAKE_BINARY_DIR}/civetweb-config.cmake) + find_package(civetweb REQUIRED) elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib") include(CivetWeb) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index bb9f7da346..94bce3c218 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -18,7 +18,7 @@ function(get_fmt) if(MINIFI_FMT_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt Fmt external lib") - include(${CMAKE_BINARY_DIR}/fmt-config.cmake) + find_package(fmt REQUIRED) elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib") include(fmt) diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index 6fb39c00c1..b2dff1892c 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -18,7 +18,7 @@ function(get_curl SOURCE_DIR BINARY_DIR) if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt libcurl external lib") - include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) + find_package(CURL REQUIRED) elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") include(BundledLibcURL) diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index 6381e6dccd..369f1285c9 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -18,7 +18,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") - include(${CMAKE_BINARY_DIR}/libxml2-config.cmake) + find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib") include(BundledLibXml2) diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index 9c8b2542d2..21d34dadb9 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -18,7 +18,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") - include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) + find_package(OpenSSL REQUIRED) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") include(BundledOpenSSL) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index a30d1215de..81a9b9afdf 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -21,7 +21,7 @@ function(get_spdlog) if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt Spdlog external lib") - include(${CMAKE_BINARY_DIR}/spdlog-config.cmake) + find_package(spdlog REQUIRED) elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") include(Spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index aae8d59e80..9b018df76e 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -18,7 +18,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR) if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt zlib external lib") - include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) + find_package(ZLIB REQUIRED) elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") include(BundledZLIB) From bfecd7ec17362e798a4036e43886fe0723a2011a Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 10 Aug 2024 00:28:46 -0700 Subject: [PATCH 14/26] Removed OPENWSMAN check; Forgot to Remove During Rebase Verified can still create MiNiFi C++ conan package successfully --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4626f4c28f..8c460aa0fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,7 +341,7 @@ include(Extensions) add_subdirectory(libminifi) -if ((ENABLE_OPENWSMAN AND ENABLE_CIVET) OR ENABLE_ALL OR ENABLE_AZURE) +if (ENABLE_ALL OR ENABLE_AZURE) include(GetLibXml2) get_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy") From 9090dcfb516681606968bccadaf4c0036134fd3d Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 10 Aug 2024 00:43:56 -0700 Subject: [PATCH 15/26] Removed OPENWSMAN tc.variable; Forgot to Remove During Rebase Verified can still create MiNiFi C++ conan package successfully. This OPENWSMAN tc.variable was related to the CMake check that is now removed. --- conanfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 96f4c929fc..8231a00a85 100644 --- a/conanfile.py +++ b/conanfile.py @@ -45,7 +45,6 @@ def generate(self): tc.variables["MINIFI_SPDLOG_SOURCE"] = "CONAN" tc.variables["SKIP_TESTS"] = "OFF" - tc.variables["ENABLE_OPENWSMAN"] = "ON" tc.variables["ENABLE_CIVET"] = "ON" tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" tc.variables["ENABLE_BZIP2"] = "ON" From df67babb5222c6e2ad8f730183d78df971937667 Mon Sep 17 00:00:00 2001 From: james94 Date: Sat, 10 Aug 2024 08:43:23 -0700 Subject: [PATCH 16/26] Fixed linter issue in conanfile.py & test_package/conanfile.py --- conanfile.py | 19 +--- test_package/.gitignore | 8 ++ test_package/CMakeLists.txt | 38 +++++++ test_package/conanfile.py | 31 ++++++ test_package/src/AbstractProcessorTest.cpp | 114 +++++++++++++++++++++ 5 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 test_package/.gitignore create mode 100644 test_package/CMakeLists.txt create mode 100644 test_package/conanfile.py create mode 100644 test_package/src/AbstractProcessorTest.cpp diff --git a/conanfile.py b/conanfile.py index 8231a00a85..f1ac9a87de 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,34 +1,25 @@ from conan import ConanFile -from conan.tools.env import VirtualRunEnv from conan.tools.cmake import CMake, CMakeToolchain from conan.tools.files import collect_libs, copy import os required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", - "catch2/3.5.4", "fmt/10.2.1", "spdlog/1.14.0") +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "catch2/3.5.4", "fmt/10.2.1", "spdlog/1.14.0") + +shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") -shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", - "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", - "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", - "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", - "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", - "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", - "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", - "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", - "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") class MiNiFiCppMain(ConanFile): name = "minifi-cpp" version = "0.99.1" license = "Apache-2.0" requires = shared_requires - settings = "os", "compiler", "build_type", "arch" + settings = ("os", "compiler", "build_type", "arch") generators = "CMakeDeps" options = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True,} + default_options = {"shared": False, "fPIC": True} exports_sources = shared_sources diff --git a/test_package/.gitignore b/test_package/.gitignore new file mode 100644 index 0000000000..8a787a020e --- /dev/null +++ b/test_package/.gitignore @@ -0,0 +1,8 @@ +cmake-build-release* +CMakeUserPresets.json +conan.lock +conanbuildinfo.txt +conanbuildinfo.cmake +conaninfo.txt +graph_info.json +build diff --git a/test_package/CMakeLists.txt b/test_package/CMakeLists.txt new file mode 100644 index 0000000000..01547802f1 --- /dev/null +++ b/test_package/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.15) +project(test-minifi-cpp CXX) + +find_package(minifi-cpp CONFIG REQUIRED) +find_package(gsl-lite CONFIG REQUIRED) +find_package(fmt CONFIG REQUIRED) +find_package(spdlog CONFIG REQUIRED) +find_package(Catch2 CONFIG REQUIRED) + +add_executable(abstract_processor_test src/AbstractProcessorTest.cpp) + +target_link_libraries(abstract_processor_test +minifi-cpp::minifi-cpp +gsl::gsl-lite +fmt::fmt +spdlog::spdlog +Catch2::Catch2WithMain +) + +list(APPEND gsl-lite_DEFINITIONS_RELEASE gsl_CONFIG_DEFAULTS_VERSION=1) + +target_compile_definitions(gsl::gsl-lite INTERFACE ${gsl-lite_DEFINITIONS_RELEASE}) + +target_include_directories(abstract_processor_test PUBLIC + ${fmt_INCLUDE_DIRS} + ${minifi-cpp_INCLUDE_DIRS} + ${minifi-cpp_INCLUDE_DIRS}/libminifi/include + ${minifi-cpp_INCLUDE_DIRS}/libminifi/test/libtest + ${minifi-cpp_INCLUDE_DIRS}/_deps/magic_enum-src/include + ${minifi-cpp_INCLUDE_DIRS}/_deps/date_src-src/include + ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-src/src + ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-build/generated-includes + ${minifi-cpp_INCLUDE_DIRS}/_deps/spdlog-src/include + ${minifi-cpp_INCLUDE_DIRS}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include + ${minifi-cpp_INCLUDE_DIRS}/extensions/standard-processors/processors +) + +add_test(NAME abstract_processor_test COMMAND abstract_processor_test) diff --git a/test_package/conanfile.py b/test_package/conanfile.py new file mode 100644 index 0000000000..068fe1683f --- /dev/null +++ b/test_package/conanfile.py @@ -0,0 +1,31 @@ +import os + +from conan import ConanFile +from conan.tools.build import can_run +from conan.tools.cmake import CMake, cmake_layout + + +class MiNiFiCppTestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + self.requires("gsl-lite/0.41.0") + self.requires("fmt/10.2.1") + self.requires("spdlog/1.14.0") + self.requires("catch2/3.5.4") + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def layout(self): + cmake_layout(self) + + def test(self): + if can_run(self): + cmd = os.path.join(self.cpp.build.bindirs[0], "abstract_processor_test") + self.run(cmd, env="conanrun") diff --git a/test_package/src/AbstractProcessorTest.cpp b/test_package/src/AbstractProcessorTest.cpp new file mode 100644 index 0000000000..99506c63d5 --- /dev/null +++ b/test_package/src/AbstractProcessorTest.cpp @@ -0,0 +1,114 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define EXTENSION_LIST "" // NOLINT(cppcoreguidelines-macro-usage) +#include +#include "unit/Catch.h" +#include "core/AbstractProcessor.h" +#include "core/PropertyDefinitionBuilder.h" + +namespace org::apache::nifi::minifi::test { + +struct AbstractProcessorTestCase1 : core::AbstractProcessor { + static constexpr auto SupportsDynamicProperties = true; + static constexpr auto SupportsDynamicRelationships = true; + static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; + static constexpr auto IsSingleThreaded = true; + static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("Property 1") + .withDefaultValue("foo") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("Property 2") + .withDefaultValue("bar") + .supportsExpressionLanguage(false) + .isRequired(false) + .build(); + static constexpr auto Properties = std::array{Property1, Property2}; + static constexpr core::RelationshipDefinition Rel1{"rel1", "rel1 description"}; + static constexpr auto Relationships = std::array{Rel1}; + AbstractProcessorTestCase1() :core::AbstractProcessor{"TestCase1"} {} + void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} + void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} +}; + +TEST_CASE("AbstractProcessor case1", "[processor][abstractprocessor][case1]") { + AbstractProcessorTestCase1 processor; + processor.initialize(); + REQUIRE(processor.getName() == "TestCase1"); + REQUIRE(processor.supportsDynamicProperties() == true); + REQUIRE(processor.supportsDynamicRelationships() == true); + REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_FORBIDDEN); + REQUIRE(processor.isSingleThreaded()); + const auto properties = processor.getProperties(); + REQUIRE(properties.size() == 2); + REQUIRE(properties.contains("Property 1")); + REQUIRE(properties.at("Property 1").supportsExpressionLanguage()); + REQUIRE(properties.at("Property 1").getRequired()); + REQUIRE(properties.contains("Property 2")); + REQUIRE(!properties.at("Property 2").supportsExpressionLanguage()); + REQUIRE(!properties.at("Property 2").getRequired()); + REQUIRE(processor.getSupportedRelationships().size() == 1); + const auto relationships = processor.getSupportedRelationships(); + REQUIRE(relationships[0].getName() == "rel1"); +} + +struct AbstractProcessorTestCase2 : core::AbstractProcessor { + static constexpr bool SupportsDynamicProperties = false; + static constexpr bool SupportsDynamicRelationships = false; + static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; + static constexpr bool IsSingleThreaded = false; + static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("prop 1") + .withDefaultValue("foo") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("prop 2") + .withDefaultValue("bar") + .supportsExpressionLanguage(false) + .isRequired(false) + .build(); + static constexpr auto Properties = std::array{Property1, Property2}; + static constexpr core::RelationshipDefinition Rel1{"Relationship1", "rel1 description"}; + static constexpr auto Relationships = std::array{Rel1}; + AbstractProcessorTestCase2() :core::AbstractProcessor{"TestCase2"} {} + void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} + void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} +}; + +TEST_CASE("AbstractProcessor case2", "[processor][abstractprocessor][case2]") { + AbstractProcessorTestCase2 processor; + processor.initialize(); + REQUIRE(processor.getName() == "TestCase2"); + REQUIRE(!processor.supportsDynamicProperties()); + REQUIRE(!processor.supportsDynamicRelationships()); + REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_REQUIRED); + REQUIRE(!processor.isSingleThreaded()); + const auto properties = processor.getProperties(); + REQUIRE(properties.size() == 2); + REQUIRE(properties.contains("prop 1")); + REQUIRE(properties.at("prop 1").supportsExpressionLanguage()); + REQUIRE(properties.at("prop 1").getRequired()); + REQUIRE(properties.contains("prop 2")); + REQUIRE(!properties.at("prop 2").supportsExpressionLanguage()); + REQUIRE(!properties.at("prop 2").getRequired()); + REQUIRE(processor.getSupportedRelationships().size() == 1); + const auto relationships = processor.getSupportedRelationships(); + REQUIRE(relationships[0].getName() == "Relationship1"); +} + +} // namespace org::apache::nifi::minifi::test From 8c5db7bc61e9b161f7ffea760cb69e1cf4b88ef2 Mon Sep 17 00:00:00 2001 From: james94 Date: Thu, 15 Aug 2024 13:15:14 -0700 Subject: [PATCH 17/26] Fixed Standalone CMake CTEST Issues, So All 275 TESTs PASS for Ubuntu22.04 Verified that I can build MiNiFi C++ in standalone CMake mode and that all 275 CTESTs PASS. Verified I can run conan to build MiNIFi C++ with some features DISABLED, so its easier to create the MiNiFi C++ conan package. In the case of building MiNiFi C++ with conan, when we run CTEST, 219 TESTs run and 217 TESTs PASS. We dont include in conanfile.py 'cmake.ctest()', so that we can manually run ctest after conan finishes building MiNiFi C++. Also removed the test_package/ folder and C++ MiNiFi C++ test for AbstractProcessorTest since I plan to later do a follow up PR that runs a new TEST that will be closer to how I plan to use MiNiFi C++ in other C++ projects. For example, in one of the follow up PRs, I plan to programmatically create an edge data pipeline that processes medical images as an example. I think these updates will enable the CI/CD to pass, but also need to double check --- cmake/BuildTests.cmake | 4 +- cmake/GetCatch2.cmake | 3 +- cmake/GetSpdlog.cmake | 2 + cmake/MiNiFiOptions.cmake | 3 +- conanfile.py | 66 ++++------ controller/tests/CMakeLists.txt | 2 +- encrypt-config/tests/CMakeLists.txt | 2 +- extensions/aws/tests/CMakeLists.txt | 2 +- extensions/azure/tests/CMakeLists.txt | 2 +- extensions/bustache/tests/CMakeLists.txt | 2 +- extensions/civetweb/tests/CMakeLists.txt | 2 +- extensions/elasticsearch/tests/CMakeLists.txt | 2 +- .../execute-process/test/CMakeLists.txt | 2 +- .../expression-language/tests/CMakeLists.txt | 4 +- extensions/grafana-loki/tests/CMakeLists.txt | 2 +- extensions/kubernetes/tests/CMakeLists.txt | 2 +- extensions/libarchive/tests/CMakeLists.txt | 2 +- extensions/librdkafka/tests/CMakeLists.txt | 2 +- extensions/lua/tests/CMakeLists.txt | 2 +- extensions/mqtt/tests/CMakeLists.txt | 2 +- extensions/opencv/tests/CMakeLists.txt | 2 +- extensions/pdh/tests/CMakeLists.txt | 2 +- extensions/procfs/tests/CMakeLists.txt | 2 +- extensions/prometheus/tests/CMakeLists.txt | 2 +- extensions/python/tests/CMakeLists.txt | 4 +- extensions/rocksdb-repos/tests/CMakeLists.txt | 2 +- extensions/script/tests/CMakeLists.txt | 2 +- extensions/sftp/tests/CMakeLists.txt | 2 +- extensions/smb/tests/CMakeLists.txt | 2 +- extensions/splunk/tests/CMakeLists.txt | 2 +- extensions/sql/tests/CMakeLists.txt | 2 +- .../standard-processors/tests/CMakeLists.txt | 4 +- extensions/systemd/tests/CMakeLists.txt | 2 +- .../windows-event-log/tests/CMakeLists.txt | 2 +- libminifi/test/flow-tests/CMakeLists.txt | 2 +- libminifi/test/integration/CMakeLists.txt | 2 +- libminifi/test/libtest/unit/CMakeLists.txt | 2 +- .../test/persistence-tests/CMakeLists.txt | 2 +- libminifi/test/schema-tests/CMakeLists.txt | 2 +- libminifi/test/unit/CMakeLists.txt | 2 +- minifi_main/tests/CMakeLists.txt | 2 +- test_package/.gitignore | 8 -- test_package/CMakeLists.txt | 38 ------ test_package/conanfile.py | 31 ----- test_package/src/AbstractProcessorTest.cpp | 114 ------------------ 45 files changed, 69 insertions(+), 278 deletions(-) delete mode 100644 test_package/.gitignore delete mode 100644 test_package/CMakeLists.txt delete mode 100644 test_package/conanfile.py delete mode 100644 test_package/src/AbstractProcessorTest.cpp diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index c18079c4e5..37e19fa106 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -77,7 +77,7 @@ function(createTests testName) target_link_libraries(${testName} ${CMAKE_DL_LIBS}) target_wholearchive_library(${testName} libminifi-unittest) - target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads Catch2::Catch2WithMain) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") @@ -89,7 +89,7 @@ function(createIntegrationTests testName) target_link_libraries(${testName} ${CMAKE_DL_LIBS}) target_wholearchive_library(${testName} libminifi-integrationtest) - target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads Catch2::Catch2WithMain) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index d30c5c8ac4..23ff9dbee7 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -19,8 +19,7 @@ function(get_catch2) if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt Catch2 external lib") find_package(Catch2 REQUIRED) - message("Catch2_INCLUDE_DIRS = ${Catch2_INCLUDE_DIRS}") - set(CATCH2_INCLUDE_DIRS "${Catch2_INCLUDE_DIRS}" CACHE STRING "" FORCE) + add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib") include(Catch2) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index 81a9b9afdf..b9f4233eff 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -22,6 +22,8 @@ function(get_spdlog) if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") message("Using Conan Packager to manage installing prebuilt Spdlog external lib") find_package(spdlog REQUIRED) + + add_library(spdlog ALIAS spdlog::spdlog) elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") include(Spdlog) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 371f3d507c..ae4a9f485a 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -140,10 +140,9 @@ add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided s add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") -add_minifi_multi_option(MINIFI_CATCH2_SOURCE "Retrieves Catch2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_SPDLOG_SOURCE "Retrieves Spdlog from provided source" "BUILD;SYSTEM;CONAN" "BUILD") - +add_minifi_multi_option(MINIFI_CATCH2_SOURCE "Retrieves Catch2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") # Docker options diff --git a/conanfile.py b/conanfile.py index f1ac9a87de..94bac3eebd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,11 +1,13 @@ from conan import ConanFile from conan.tools.cmake import CMake, CMakeToolchain from conan.tools.files import collect_libs, copy +from conan.errors import ConanException import os +import shutil required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "catch2/3.5.4", "fmt/10.2.1", "spdlog/1.14.0") +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") @@ -15,7 +17,7 @@ class MiNiFiCppMain(ConanFile): version = "0.99.1" license = "Apache-2.0" requires = shared_requires - settings = ("os", "compiler", "build_type", "arch") + settings = "os", "compiler", "build_type", "arch" generators = "CMakeDeps" options = {"shared": [True, False], "fPIC": [True, False]} @@ -31,55 +33,18 @@ def generate(self): tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" - tc.variables["MINIFI_CATCH2_SOURCE"] = "CONAN" tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" tc.variables["MINIFI_SPDLOG_SOURCE"] = "CONAN" + tc.variables["MINIFI_CATCH2_SOURCE"] = "CONAN" tc.variables["SKIP_TESTS"] = "OFF" tc.variables["ENABLE_CIVET"] = "ON" tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" - tc.variables["ENABLE_BZIP2"] = "ON" - tc.variables["ENABLE_ROCKSDB"] = "ON" - tc.variables["BUILD_ROCKSDB"] = "ON" - tc.variables["ENABLE_LIBARCHIVE"] = "OFF" tc.variables["ENABLE_AWS"] = "OFF" - - tc.variables["ENABLE_OPC"] = "OFF" - - if self.settings.os == "Windows": - tc.variables["ENABLE_WEL"] = "OFF" - tc.variables["ENABLE_PDH"] = "OFF" - tc.variables["ENABLE_SMB"] = "OFF" - elif self.settings.os == "Linux": - tc.variables["ENABLE_SYSTEMD"] = "OFF" - tc.variables["ENABLE_PROCFS"] = "OFF" - - tc.variables["ENABLE_LZMA"] = "OFF" - tc.variables["ENABLE_GPS"] = "OFF" - tc.variables["ENABLE_COAP"] = "OFF" tc.variables["ENABLE_SQL"] = "OFF" - tc.variables["ENABLE_MQTT"] = "OFF" - tc.variables["ENABLE_PCAP"] = "OFF" - tc.variables["ENABLE_LIBRDKAFKA"] = "OFF" - tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" - tc.variables["ENABLE_PYTHON_SCRIPTING"] = "OFF" - tc.variables["ENABLE_SENSORS"] = "OFF" - tc.variables["ENABLE_USB_CAMERA"] = "OFF" - tc.variables["ENABLE_OPENCV"] = "OFF" - tc.variables["ENABLE_BUSTACHE"] = "OFF" - tc.variables["ENABLE_SFTP"] = "OFF" - tc.variables["ENABLE_AZURE"] = "OFF" - tc.variables["ENABLE_ENCRYPT_CONFIG"] = "OFF" - tc.variables["ENABLE_SPLUNK"] = "OFF" - tc.variables["ENABLE_ELASTICSEARCH"] = "OFF" tc.variables["ENABLE_GCP"] = "OFF" - tc.variables["ENABLE_KUBERNETES"] = "OFF" - tc.variables["ENABLE_TEST_PROCESSORS"] = "OFF" - tc.variables["ENABLE_PROMETHEUS"] = "OFF" - tc.variables["ENABLE_GRAFANA_LOKI"] = "OFF" - tc.variables["ENABLE_GRPC_FOR_LOKI"] = "OFF" - tc.variables["ENABLE_CONTROLLER"] = "OFF" + tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" tc.generate() @@ -88,15 +53,32 @@ def build(self): cmake.configure() cmake.build() + def overwrite_libfile(self, oldfile, newfile): + print("Copying {} to {}".format(oldfile, newfile)) + if os.path.exists(oldfile): + try: + if os.path.exists(newfile): + os.remove(newfile) + + shutil.copy2(oldfile, newfile) + except Exception as e: + raise ConanException(f"Error copying {newfile}: {e}") + else: + raise ConanException(f"Error: {oldfile} does not exist") + def package(self): cmake = CMake(self) + cmake.install() include_dir = os.path.join(self.source_folder) built_dir = os.path.join(self.source_folder, self.folders.build) copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) - cmake.install() + + minifi_py_ext_oldfile = os.path.join(self.package_folder, "lib", "libminifi-python-script-extension.so") + minifi_py_ext_copynewfile = os.path.join(self.package_folder, "lib", "libminifi_native.so") + self.overwrite_libfile(minifi_py_ext_oldfile, minifi_py_ext_copynewfile) def package_info(self): self.cpp_info.libs = collect_libs(self, folder=os.path.join(self.package_folder, "lib")) diff --git a/controller/tests/CMakeLists.txt b/controller/tests/CMakeLists.txt index 056969fa60..c5fca51a6f 100644 --- a/controller/tests/CMakeLists.txt +++ b/controller/tests/CMakeLists.txt @@ -36,7 +36,7 @@ foreach(testfile ${CONTROLLER_TESTS}) createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) + target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR CONTROLLER_TEST_COUNT "${CONTROLLER_TEST_COUNT}+1") diff --git a/encrypt-config/tests/CMakeLists.txt b/encrypt-config/tests/CMakeLists.txt index 91d8e39dd3..175e70cff2 100644 --- a/encrypt-config/tests/CMakeLists.txt +++ b/encrypt-config/tests/CMakeLists.txt @@ -31,7 +31,7 @@ foreach(testfile ${ENCRYPT_CONFIG_TESTS}) createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) + target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR ENCRYPT_CONFIG_TEST_COUNT "${ENCRYPT_CONFIG_TEST_COUNT}+1") diff --git a/extensions/aws/tests/CMakeLists.txt b/extensions/aws/tests/CMakeLists.txt index 7bc116b616..d26c0ebee9 100644 --- a/extensions/aws/tests/CMakeLists.txt +++ b/extensions/aws/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${AWS_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-aws) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/azure/tests/CMakeLists.txt b/extensions/azure/tests/CMakeLists.txt index 9bce4a99d4..730cf34074 100644 --- a/extensions/azure/tests/CMakeLists.txt +++ b/extensions/azure/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${AZURE_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") target_compile_features(${testfilename} PRIVATE cxx_std_14) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-azure) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/bustache/tests/CMakeLists.txt b/extensions/bustache/tests/CMakeLists.txt index 5ec7664f38..4f493df658 100644 --- a/extensions/bustache/tests/CMakeLists.txt +++ b/extensions/bustache/tests/CMakeLists.txt @@ -30,7 +30,7 @@ FOREACH(testfile ${BUSTACHE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR BUSTACHE-EXTENSIONS_TEST_COUNT "${BUSTACHE-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/civetweb/tests/CMakeLists.txt b/extensions/civetweb/tests/CMakeLists.txt index 23dceab36a..95f7af7d87 100644 --- a/extensions/civetweb/tests/CMakeLists.txt +++ b/extensions/civetweb/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR CIVETWEB-EXTENSIONS_TEST_COUNT "${CIVETWEB-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) # Copy test resources diff --git a/extensions/elasticsearch/tests/CMakeLists.txt b/extensions/elasticsearch/tests/CMakeLists.txt index 82a7ff6432..58a2120131 100644 --- a/extensions/elasticsearch/tests/CMakeLists.txt +++ b/extensions/elasticsearch/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${ELASTICSEARCH_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-elasticsearch) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/execute-process/test/CMakeLists.txt b/extensions/execute-process/test/CMakeLists.txt index 0467357804..05d5f93480 100644 --- a/extensions/execute-process/test/CMakeLists.txt +++ b/extensions/execute-process/test/CMakeLists.txt @@ -22,7 +22,7 @@ set_target_properties(EchoParameters PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAK add_minifi_executable(ExecuteProcessTests ExecuteProcessTests.cpp) -target_link_libraries(ExecuteProcessTests minifi-execute-process minifi-standard-processors Catch2::Catch2WithMain) +target_link_libraries(ExecuteProcessTests minifi-execute-process minifi-standard-processors Catch2WithMain) target_include_directories(ExecuteProcessTests PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test") add_test(NAME ExecuteProcessTests COMMAND ExecuteProcessTests WORKING_DIRECTORY ${TEST_DIR}) createTests(ExecuteProcessTests) diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt index c5e08eca89..2f96ee3df2 100644 --- a/extensions/expression-language/tests/CMakeLists.txt +++ b/extensions/expression-language/tests/CMakeLists.txt @@ -29,7 +29,7 @@ FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") @@ -52,7 +52,7 @@ FOREACH(testfile ${INT_EXPRESSION_LANGUAGE_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createIntegrationTests(${testfilename}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") diff --git a/extensions/grafana-loki/tests/CMakeLists.txt b/extensions/grafana-loki/tests/CMakeLists.txt index d32fea604f..8aa24fd887 100644 --- a/extensions/grafana-loki/tests/CMakeLists.txt +++ b/extensions/grafana-loki/tests/CMakeLists.txt @@ -34,7 +34,7 @@ FOREACH(testfile ${GRAFANA_LOKI_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-grafana-loki) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/kubernetes/tests/CMakeLists.txt b/extensions/kubernetes/tests/CMakeLists.txt index 0c2c0d295e..d5dd9abcab 100644 --- a/extensions/kubernetes/tests/CMakeLists.txt +++ b/extensions/kubernetes/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(TEST_FILE ${KUBERNETES_UNIT_TESTS}) target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/kubernetes") target_link_libraries(${TEST_TARGET} minifi-kubernetes-extensions) - target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR KUBERNETES_UNIT_TEST_COUNT "${KUBERNETES_UNIT_TEST_COUNT}+1") diff --git a/extensions/libarchive/tests/CMakeLists.txt b/extensions/libarchive/tests/CMakeLists.txt index 1358371492..5c1ca342fe 100644 --- a/extensions/libarchive/tests/CMakeLists.txt +++ b/extensions/libarchive/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${ARCHIVE_INTEGRATION_TESTS}) target_link_libraries(${testfilename} minifi-archive-extensions) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/librdkafka/tests/CMakeLists.txt b/extensions/librdkafka/tests/CMakeLists.txt index c8d0798951..7f8053884c 100644 --- a/extensions/librdkafka/tests/CMakeLists.txt +++ b/extensions/librdkafka/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${KAFKA_INTEGRATION_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/librdkafka") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-rdkafka-extensions) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/lua/tests/CMakeLists.txt b/extensions/lua/tests/CMakeLists.txt index 9922d94e35..38d6979906 100644 --- a/extensions/lua/tests/CMakeLists.txt +++ b/extensions/lua/tests/CMakeLists.txt @@ -27,7 +27,7 @@ FOREACH(testfile ${EXECUTESCRIPT_LUA_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/lua") target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${SOL2_INCLUDE_DIR}") target_link_libraries(${testfilename} minifi-lua-script-extension minifi-script-extension) - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/mqtt/tests/CMakeLists.txt b/extensions/mqtt/tests/CMakeLists.txt index 913eb1ef77..d5b408a64a 100644 --- a/extensions/mqtt/tests/CMakeLists.txt +++ b/extensions/mqtt/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${MQTT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../../libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-mqtt-extensions) target_link_libraries(${testfilename} minifi-standard-processors) add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/opencv/tests/CMakeLists.txt b/extensions/opencv/tests/CMakeLists.txt index cc34d8550f..d095c7e9a2 100644 --- a/extensions/opencv/tests/CMakeLists.txt +++ b/extensions/opencv/tests/CMakeLists.txt @@ -31,6 +31,6 @@ FOREACH(testfile ${OPENCV_TESTS}) createTests("${testfilename}") MATH(EXPR OPENCV_TEST_COUNT "${OPENCV_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) ENDFOREACH() message("-- Finished building ${OPENCV_TEST_COUNT} OpenCV related test file(s)...") diff --git a/extensions/pdh/tests/CMakeLists.txt b/extensions/pdh/tests/CMakeLists.txt index 5ba1b57585..63686553fa 100644 --- a/extensions/pdh/tests/CMakeLists.txt +++ b/extensions/pdh/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${PDH_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/pdh") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-pdh) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PDH_TEST_COUNT "${PDH_TEST_COUNT}+1") diff --git a/extensions/procfs/tests/CMakeLists.txt b/extensions/procfs/tests/CMakeLists.txt index 060f51dadd..ad51a869d3 100644 --- a/extensions/procfs/tests/CMakeLists.txt +++ b/extensions/procfs/tests/CMakeLists.txt @@ -27,7 +27,7 @@ FOREACH(testfile ${PROCFS_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/procfs") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-procfs) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PROCFS_TEST_COUNT "${PROCFS_TEST_COUNT}+1") diff --git a/extensions/prometheus/tests/CMakeLists.txt b/extensions/prometheus/tests/CMakeLists.txt index ebe0db595d..29b3185c43 100644 --- a/extensions/prometheus/tests/CMakeLists.txt +++ b/extensions/prometheus/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${PROMETHEUS_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/prometheus") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-prometheus) MATH(EXPR PROMETHEUS_TEST_COUNT "${PROMETHEUS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") diff --git a/extensions/python/tests/CMakeLists.txt b/extensions/python/tests/CMakeLists.txt index 3c5082aacd..06cc89e44a 100644 --- a/extensions/python/tests/CMakeLists.txt +++ b/extensions/python/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${EXECUTESCRIPT_PYTHON_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2::Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -44,7 +44,7 @@ FOREACH(testfile ${EXECUTEPYTHONPROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2::Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/rocksdb-repos/tests/CMakeLists.txt b/extensions/rocksdb-repos/tests/CMakeLists.txt index 18088e0c91..53b47dbbcd 100644 --- a/extensions/rocksdb-repos/tests/CMakeLists.txt +++ b/extensions/rocksdb-repos/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${ROCKSDB_UNIT_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/rocksdb-repos/") target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${ROCKSDB_THIRDPARTY_ROOT}/include") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-rocksdb-repos) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR ROCKSDB_TEST_COUNT "${ROCKSDB_TEST_COUNT}+1") diff --git a/extensions/script/tests/CMakeLists.txt b/extensions/script/tests/CMakeLists.txt index 5c88322264..2157777555 100644 --- a/extensions/script/tests/CMakeLists.txt +++ b/extensions/script/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${EXECUTE_SCRIPT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-script-extension) MATH(EXPR EXECUTE_SCRIPT_TEST_COUNT "${EXECUTE_SCRIPT_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/sftp/tests/CMakeLists.txt b/extensions/sftp/tests/CMakeLists.txt index 7276b275b5..a8767d7bcb 100644 --- a/extensions/sftp/tests/CMakeLists.txt +++ b/extensions/sftp/tests/CMakeLists.txt @@ -40,7 +40,7 @@ if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE target_include_directories(${testfilename} BEFORE PRIVATE ./tools) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain core-minifi sftp-test-tools) + target_link_libraries(${testfilename} Catch2WithMain core-minifi sftp-test-tools) target_link_libraries(${testfilename} minifi-sftp) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/smb/tests/CMakeLists.txt b/extensions/smb/tests/CMakeLists.txt index 1a5ab5eec5..d1d02b6953 100644 --- a/extensions/smb/tests/CMakeLists.txt +++ b/extensions/smb/tests/CMakeLists.txt @@ -26,7 +26,7 @@ FOREACH(testfile ${SMB_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/smb") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-smb) target_link_libraries(${testfilename} minifi-standard-processors Netapi32) MATH(EXPR SMB_TEST_COUNT "${SMB_TEST_COUNT}+1") diff --git a/extensions/splunk/tests/CMakeLists.txt b/extensions/splunk/tests/CMakeLists.txt index 313844c52d..f00d2740b4 100644 --- a/extensions/splunk/tests/CMakeLists.txt +++ b/extensions/splunk/tests/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${SPLUNK_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-splunk) target_link_libraries(${testfilename} minifi-civet-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/sql/tests/CMakeLists.txt b/extensions/sql/tests/CMakeLists.txt index 2996cbad1e..81184f0664 100644 --- a/extensions/sql/tests/CMakeLists.txt +++ b/extensions/sql/tests/CMakeLists.txt @@ -34,7 +34,7 @@ foreach(testfile ${SQL_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-sql-mocks) target_link_libraries(${testfilename} minifi-sql) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/standard-processors/tests/CMakeLists.txt b/extensions/standard-processors/tests/CMakeLists.txt index a17229003f..4eff161456 100644 --- a/extensions/standard-processors/tests/CMakeLists.txt +++ b/extensions/standard-processors/tests/CMakeLists.txt @@ -36,7 +36,7 @@ FOREACH(testfile ${PROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE JOLT_TESTS_DIR="${jolt_tests_SOURCE_DIR}/jolt-core/src/test/resources/json/shiftr") @@ -66,7 +66,7 @@ FOREACH(testfile ${PROCESSOR_INTEGRATION_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE ./include) createIntegrationTests("${testfilename}") target_link_libraries(${testfilename}) - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") diff --git a/extensions/systemd/tests/CMakeLists.txt b/extensions/systemd/tests/CMakeLists.txt index 77160a591b..9d245ea236 100644 --- a/extensions/systemd/tests/CMakeLists.txt +++ b/extensions/systemd/tests/CMakeLists.txt @@ -26,7 +26,7 @@ function(add_systemd_test testfile) target_link_libraries(${TEST_TARGET} minifi-standard-processors) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") - target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2WithMain) endfunction() add_systemd_test("ConsumeJournaldTest.cpp") diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 1939cd3553..07c40712f5 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -36,7 +36,7 @@ FOREACH(testfile ${WEL_TESTS}) createTests("${testfilename}") target_link_libraries(${testfilename} minifi-wel) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries (${testfilename} ${LIBMINIFI} Catch2::Catch2WithMain) + target_link_libraries (${testfilename} ${LIBMINIFI} Catch2WithMain) MATH(EXPR WEL_TEST_COUNT "${WEL_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/libminifi/test/flow-tests/CMakeLists.txt b/libminifi/test/flow-tests/CMakeLists.txt index 9111ffdefc..67845af565 100644 --- a/libminifi/test/flow-tests/CMakeLists.txt +++ b/libminifi/test/flow-tests/CMakeLists.txt @@ -31,7 +31,7 @@ FOREACH(testfile ${FLOW_TESTS}) target_link_libraries(${testfilename} minifi-rocksdb-repos) endif() - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR FLOW_TEST_COUNT "${FLOW_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") ENDFOREACH() diff --git a/libminifi/test/integration/CMakeLists.txt b/libminifi/test/integration/CMakeLists.txt index c52f40cf9e..bd7bf697ad 100644 --- a/libminifi/test/integration/CMakeLists.txt +++ b/libminifi/test/integration/CMakeLists.txt @@ -28,7 +28,7 @@ FOREACH(testfile ${INTEGRATION_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} Catch2::Catch2WithMain Threads::Threads) + target_link_libraries(${testfilename} Catch2WithMain Threads::Threads) target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") if(NOT ${testfilename} IN_LIST TESTS_WITH_LIBARCHIVE_DEPENDENCY) add_test(NAME "${testfilename}" COMMAND "${testfilename}") diff --git a/libminifi/test/libtest/unit/CMakeLists.txt b/libminifi/test/libtest/unit/CMakeLists.txt index dccb78a9ac..3b90524e90 100644 --- a/libminifi/test/libtest/unit/CMakeLists.txt +++ b/libminifi/test/libtest/unit/CMakeLists.txt @@ -18,7 +18,7 @@ # file(GLOB TEST_BASE_SOURCES "*.cpp") add_minifi_library(libminifi-unittest STATIC ${TEST_BASE_SOURCES}) -target_link_libraries(libminifi-unittest core-minifi Catch2::Catch2WithMain) +target_link_libraries(libminifi-unittest core-minifi Catch2WithMain) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") if(WIN32) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") diff --git a/libminifi/test/persistence-tests/CMakeLists.txt b/libminifi/test/persistence-tests/CMakeLists.txt index c2b51a2afe..c2913bd6d6 100644 --- a/libminifi/test/persistence-tests/CMakeLists.txt +++ b/libminifi/test/persistence-tests/CMakeLists.txt @@ -32,7 +32,7 @@ FOREACH(testfile ${PERSISTENCE_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-rocksdb-repos) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR PERSISTENCE_TEST_COUNT "${PERSISTENCE_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/libminifi/test/schema-tests/CMakeLists.txt b/libminifi/test/schema-tests/CMakeLists.txt index 0a53ed1f10..a5befa0829 100644 --- a/libminifi/test/schema-tests/CMakeLists.txt +++ b/libminifi/test/schema-tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${SCHEMA_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} nlohmann_json_schema_validator) MATH(EXPR SCHEMA_TEST_COUNT "${SCHEMA_TEST_COUNT}+1") diff --git a/libminifi/test/unit/CMakeLists.txt b/libminifi/test/unit/CMakeLists.txt index 94308c0a49..08e93d4cab 100644 --- a/libminifi/test/unit/CMakeLists.txt +++ b/libminifi/test/unit/CMakeLists.txt @@ -30,7 +30,7 @@ FOREACH(testfile ${UNIT_TESTS}) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2::Catch2WithMain) + target_link_libraries(${testfilename} Catch2WithMain) MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") ENDFOREACH() diff --git a/minifi_main/tests/CMakeLists.txt b/minifi_main/tests/CMakeLists.txt index 12afbf7b72..7620162ec5 100644 --- a/minifi_main/tests/CMakeLists.txt +++ b/minifi_main/tests/CMakeLists.txt @@ -23,7 +23,7 @@ FOREACH(TEST_FILE ${MINIFI_MAIN_UNIT_TESTS}) get_filename_component(TEST_TARGET "${TEST_FILE}" NAME_WE) add_minifi_executable("${TEST_TARGET}" "${TEST_FILE}" "../TableFormatter.cpp") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/minifi_main") - target_link_libraries(${TEST_TARGET} core-minifi Catch2::Catch2WithMain) + target_link_libraries(${TEST_TARGET} core-minifi Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR MINIFI_MAIN_UNIT_TEST_COUNT "${MINIFI_MAIN_UNIT_TEST_COUNT}+1") diff --git a/test_package/.gitignore b/test_package/.gitignore deleted file mode 100644 index 8a787a020e..0000000000 --- a/test_package/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -cmake-build-release* -CMakeUserPresets.json -conan.lock -conanbuildinfo.txt -conanbuildinfo.cmake -conaninfo.txt -graph_info.json -build diff --git a/test_package/CMakeLists.txt b/test_package/CMakeLists.txt deleted file mode 100644 index 01547802f1..0000000000 --- a/test_package/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -cmake_minimum_required(VERSION 3.15) -project(test-minifi-cpp CXX) - -find_package(minifi-cpp CONFIG REQUIRED) -find_package(gsl-lite CONFIG REQUIRED) -find_package(fmt CONFIG REQUIRED) -find_package(spdlog CONFIG REQUIRED) -find_package(Catch2 CONFIG REQUIRED) - -add_executable(abstract_processor_test src/AbstractProcessorTest.cpp) - -target_link_libraries(abstract_processor_test -minifi-cpp::minifi-cpp -gsl::gsl-lite -fmt::fmt -spdlog::spdlog -Catch2::Catch2WithMain -) - -list(APPEND gsl-lite_DEFINITIONS_RELEASE gsl_CONFIG_DEFAULTS_VERSION=1) - -target_compile_definitions(gsl::gsl-lite INTERFACE ${gsl-lite_DEFINITIONS_RELEASE}) - -target_include_directories(abstract_processor_test PUBLIC - ${fmt_INCLUDE_DIRS} - ${minifi-cpp_INCLUDE_DIRS} - ${minifi-cpp_INCLUDE_DIRS}/libminifi/include - ${minifi-cpp_INCLUDE_DIRS}/libminifi/test/libtest - ${minifi-cpp_INCLUDE_DIRS}/_deps/magic_enum-src/include - ${minifi-cpp_INCLUDE_DIRS}/_deps/date_src-src/include - ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-src/src - ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-build/generated-includes - ${minifi-cpp_INCLUDE_DIRS}/_deps/spdlog-src/include - ${minifi-cpp_INCLUDE_DIRS}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include - ${minifi-cpp_INCLUDE_DIRS}/extensions/standard-processors/processors -) - -add_test(NAME abstract_processor_test COMMAND abstract_processor_test) diff --git a/test_package/conanfile.py b/test_package/conanfile.py deleted file mode 100644 index 068fe1683f..0000000000 --- a/test_package/conanfile.py +++ /dev/null @@ -1,31 +0,0 @@ -import os - -from conan import ConanFile -from conan.tools.build import can_run -from conan.tools.cmake import CMake, cmake_layout - - -class MiNiFiCppTestConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" - test_type = "explicit" - - def requirements(self): - self.requires(self.tested_reference_str) - self.requires("gsl-lite/0.41.0") - self.requires("fmt/10.2.1") - self.requires("spdlog/1.14.0") - self.requires("catch2/3.5.4") - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - - def layout(self): - cmake_layout(self) - - def test(self): - if can_run(self): - cmd = os.path.join(self.cpp.build.bindirs[0], "abstract_processor_test") - self.run(cmd, env="conanrun") diff --git a/test_package/src/AbstractProcessorTest.cpp b/test_package/src/AbstractProcessorTest.cpp deleted file mode 100644 index 99506c63d5..0000000000 --- a/test_package/src/AbstractProcessorTest.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#define EXTENSION_LIST "" // NOLINT(cppcoreguidelines-macro-usage) -#include -#include "unit/Catch.h" -#include "core/AbstractProcessor.h" -#include "core/PropertyDefinitionBuilder.h" - -namespace org::apache::nifi::minifi::test { - -struct AbstractProcessorTestCase1 : core::AbstractProcessor { - static constexpr auto SupportsDynamicProperties = true; - static constexpr auto SupportsDynamicRelationships = true; - static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; - static constexpr auto IsSingleThreaded = true; - static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("Property 1") - .withDefaultValue("foo") - .supportsExpressionLanguage(true) - .isRequired(true) - .build(); - static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("Property 2") - .withDefaultValue("bar") - .supportsExpressionLanguage(false) - .isRequired(false) - .build(); - static constexpr auto Properties = std::array{Property1, Property2}; - static constexpr core::RelationshipDefinition Rel1{"rel1", "rel1 description"}; - static constexpr auto Relationships = std::array{Rel1}; - AbstractProcessorTestCase1() :core::AbstractProcessor{"TestCase1"} {} - void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} - void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} -}; - -TEST_CASE("AbstractProcessor case1", "[processor][abstractprocessor][case1]") { - AbstractProcessorTestCase1 processor; - processor.initialize(); - REQUIRE(processor.getName() == "TestCase1"); - REQUIRE(processor.supportsDynamicProperties() == true); - REQUIRE(processor.supportsDynamicRelationships() == true); - REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_FORBIDDEN); - REQUIRE(processor.isSingleThreaded()); - const auto properties = processor.getProperties(); - REQUIRE(properties.size() == 2); - REQUIRE(properties.contains("Property 1")); - REQUIRE(properties.at("Property 1").supportsExpressionLanguage()); - REQUIRE(properties.at("Property 1").getRequired()); - REQUIRE(properties.contains("Property 2")); - REQUIRE(!properties.at("Property 2").supportsExpressionLanguage()); - REQUIRE(!properties.at("Property 2").getRequired()); - REQUIRE(processor.getSupportedRelationships().size() == 1); - const auto relationships = processor.getSupportedRelationships(); - REQUIRE(relationships[0].getName() == "rel1"); -} - -struct AbstractProcessorTestCase2 : core::AbstractProcessor { - static constexpr bool SupportsDynamicProperties = false; - static constexpr bool SupportsDynamicRelationships = false; - static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; - static constexpr bool IsSingleThreaded = false; - static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("prop 1") - .withDefaultValue("foo") - .supportsExpressionLanguage(true) - .isRequired(true) - .build(); - static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("prop 2") - .withDefaultValue("bar") - .supportsExpressionLanguage(false) - .isRequired(false) - .build(); - static constexpr auto Properties = std::array{Property1, Property2}; - static constexpr core::RelationshipDefinition Rel1{"Relationship1", "rel1 description"}; - static constexpr auto Relationships = std::array{Rel1}; - AbstractProcessorTestCase2() :core::AbstractProcessor{"TestCase2"} {} - void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} - void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} -}; - -TEST_CASE("AbstractProcessor case2", "[processor][abstractprocessor][case2]") { - AbstractProcessorTestCase2 processor; - processor.initialize(); - REQUIRE(processor.getName() == "TestCase2"); - REQUIRE(!processor.supportsDynamicProperties()); - REQUIRE(!processor.supportsDynamicRelationships()); - REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_REQUIRED); - REQUIRE(!processor.isSingleThreaded()); - const auto properties = processor.getProperties(); - REQUIRE(properties.size() == 2); - REQUIRE(properties.contains("prop 1")); - REQUIRE(properties.at("prop 1").supportsExpressionLanguage()); - REQUIRE(properties.at("prop 1").getRequired()); - REQUIRE(properties.contains("prop 2")); - REQUIRE(!properties.at("prop 2").supportsExpressionLanguage()); - REQUIRE(!properties.at("prop 2").getRequired()); - REQUIRE(processor.getSupportedRelationships().size() == 1); - const auto relationships = processor.getSupportedRelationships(); - REQUIRE(relationships[0].getName() == "Relationship1"); -} - -} // namespace org::apache::nifi::minifi::test From 2fff2628f0ed35655d7802dc5534128beadef68a Mon Sep 17 00:00:00 2001 From: james94 Date: Mon, 26 Aug 2024 01:04:04 -0700 Subject: [PATCH 18/26] Added Conan Build MiNiFi & Create MiNiFi Conan Package Section to README The Build MiNiFi & Create MiNiFi Conan Package section in the README points to a CONAN.md file that goes step by step on using the conan commands to compile MiNiFi and create a MiNiFi conan package. --- CONAN.md | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 9 ++++ 2 files changed, 131 insertions(+) create mode 100644 CONAN.md diff --git a/CONAN.md b/CONAN.md new file mode 100644 index 0000000000..adaf10d02d --- /dev/null +++ b/CONAN.md @@ -0,0 +1,122 @@ + + +# Apache NiFi - MiNiFi - C++ Conan Build Guide + +We will walk through the steps to build MiNiFi using conan version 2 that comes with CMake integration. We will also go through the process of creating a MiNiFi conan package for easier integration into other C++ projects. We will start with a discussion on building MiNiFi using the standalone CMake approach and the conan approach. After we have built MiNiFi and created a MiNiFi conan package, we will conclude by elaborating on some of the benefits that can come with integrating a MiNiFi conan package into new or existing C++ infrastructure with respect to real-time robotic systems. + +## Table of Contents + +- [Apache NiFi - MiNiFi - C++ Conan Build Guide](#apache-nifi---minifi---c---conan-build-guide) + - [Table of Contents](#table-of-contents) + - [Description](#description) + - [Build MiNiFi C++ with Conan](#build-minifi-c---with-conan) + - [Create MiNiFi C++ Conan Package](#create-minifi-c---conan-package) + - [Conclusion](#conclusion) + +## Description + +Apache NiFi MiNiFi C++ is normally built using standalone CMake. Similarly, we can also create a MiNiFi C++ package using make package. However, each time we build MiNiFi from +source, it uses CMake to install the third party external libraries that are enabled by downloading the source code and then building each library until it is available to use toward building MiNiFi. If a change is made toward MiNiFi's CMake code in regards to one or more of the external libraries, then potentially the long process of installing one or more of the externl library(ies) happens again before the MiNiFi source code is built. Another way we can build MiNiFi and create MiNiFi packages without running into having to rebuild its external libraries is using conan. Conan provides some great features toward managing to install prebuilt conan packages, which for our case is the external libraries that are needed to then build MiNiFi. We will walk through the steps to build the MiNiFi project and create a MiNiFi conan package. + +## Build MiNiFi C++ with Conan + +To build MiNiFi using conan, first we install conan version 2, then we create a **default** conan profile that will later be ignored for our custom conan profile, create a MINIFI_HOME environment variable, then we install prebuilt conan packages representing the MiNiFi external libraries and finally we compile MiNiFi. + +~~~bash +sudo pip install --force-reinstall -v "conan==2.0.17" + +# create a "default" conan profile, so conan has it on record in ~/.conan2/, before using your own custom profile. +conan profile detect + +# conanfile.py is in root dir of MiNiFi C++ project +cd $HOME/nifi-minifi-cpp + +# create MINIFI_HOME env variable for binary executable minifi +export MINIFI_HOME=$(pwd) + +# install conan packages for MiNiFi C++ using conanfile.py invoking Conan +# since we created default profile earlier, we can override it with our own minifi profile +conan install . --build=missing --output-folder=build_conan -pr=etc/conan/profiles/release-linux + +# build MiNiFi C++ using conanfile.py invoking Conan & CMake +conan build . --output-folder=build_conan -pr=etc/conan/profiles/release-linux +~~~ + +- **NOTE**: After building MiNiFi, we must have the MINIFI_HOME environment variable created in order to successfully run the minifi binary executable. + +- **NOTE**: When we install the prebuilt conan package representing the MiNiFi third party libraries, we add the `--build=missing` in case some of the prebuilt missing conan packages are not found from conancenter, jfrog, or one of our conan repositories, then conan will build the conan packages from source. We can upload new prebuilt conan packages by **[conan upload](https://docs.conan.io/2/reference/commands/upload.html)**. + +- **NOTE**: When we tell conan to build MiNiFi, conan first installs prebuilt conan packages and then it compiles MiNiFi from source, so we have the MiNiFi library and its binary executable available. + +### Run MiNiFi CTESTs from Build Folder + +~~~bash +pushd build_conan/ +ctest +popd # build_conan/ +~~~ + +### Run MiNiFi Executable + +~~~bash +# verify we can run minifi binary executable +./build_conan/bin/minifi +~~~ + +## Create MiNiFi C++ Conan Package + +To create a MiNiFi package, we will follow the similar steps we took to build MiNiFi. Then right after we install the prebuilt conan packages representing MiNiFi's external libraries, we create that MiNiFi conan package. + +~~~bash +# make sure to install conan2 for your environment +sudo pip install --force-reinstall -v "conan==2.0.17" + +# create a "default" conan profile, so conan has it on record, before using your own custom profile. Gets created in ~/.conan2/ +conan profile detect + +# conanfile.py is in root dir of MiNiFi C++ project +cd $HOME/nifi-minifi-cpp + +# create MINIFI_HOME env variable for binary executable minifi +export MINIFI_HOME=$(pwd) + +# install conan packages for MiNiFi C++ using conanfile.py invoking Conan +# since we created default profile earlier, we can override it with our own minifi profile +# make sure path is correct +conan install . --build=missing --output-folder=build_conan -pr=etc/conan/profiles/release-linux + +# create MiNiFi C++ conan package using conanfile.py invoking Conan & CMake +conan create . --user=minifi --channel=develop -pr=etc/conan/profiles/release-linux +~~~ + +- **NOTE**: When we tell conan to create the MiNiFi conan package, conan first installs prebuilt conan packages, then it compiles MiNiFi from source inside `~/.conan2/p/b/minif/b`, and then it copies over MiNiFi's libraries and its binary executables into the conan package folder `~/.conan2/p/b/minif/p`. Once we have the MiNiFi conan package, we can integrate it into other C++ infrastructure using CMake. + +### Run MiNiFi CTESTs from Its Conan Package + +~~~bash +# Example of minif conan package folder name: minifbd17f6a02da35 +pushd ~/.conan2/p/b/minif/b +ctest +popd # build_conan/ +~~~ + +## Conclusion + +To have a more consistent quick build process for MiNiFi, we can use conan version 2. When we use conan with MiNiFi's build process, we can install prebuilt conan packages that can represent MiNiFi's external libraries and then build MiNiFi quickly. We also have a bit more control over when we want to make updates to MiNiFi's external libraries that impact their build and then re-build them as prebuilt conan packages. So these external library conan packages can be used toward building MiNiFi again without having to rebuild those external libraries while rebuilding MiNiFi. Therefore, we can focus mainly on configuring the way we want MiNiFi to build without having to worry about MiNiFi's external libraries needing to be rebuilt again. Similarly, if we create a MiNiFi conan package, it will install the prebuilt conan packages, build MiNiFi from source and then create the MiNiFi package. + +There are multiple benefits of having MiNiFi prebuilt conan packages. We can upload these MiNiFi conan packages to a conan repository like jfrog for version management. We can easily integrate MiNiFi's edge data pipeline features into other C++ software infrastructure using conan's CMake support. We can still use MiNiFi for edge data collection from the IoT devices embedded on robotic systems. We can integrate MiNiFi into self-driving cars (sensor examples: cameras, lidar, radar, inertial measurement unit (IMU), electronic speed controller (ESC), steering servo, etc), into medical imaging robots (sensor examples: depth cameras, ultrasound, gamma detector, force/torque sensor, joint position sensor, etc) or some other real-time robotic system. + +By leveraging MiNiFi as a conan package, we can leverage MiNiFi that comes with the best practices of building data pipelines from NiFi and bring them into existing C++ real-time robotics infrastructure. Some teams across companies typically have their own custom edge data pipelines that process data for the different events to eventually perform actions on that data. As an alternative to all these companies and their teams having their own custom edge data pipeline libraries, MiNiFi C++, which is like a headless NiFi, can provide a more consistent standard approach for team's to build edge pipelines. Through all stages of the edge data pipelines, MiNiFi can still provide telemetry to NiFi instances running in the cloud. diff --git a/README.md b/README.md index 13c83f08f4..b80108b8df 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,8 @@ advanced features. ### Building +#### Build MiNiFi using Standalone CMake + - From your source checkout, create a directory to perform the build (e.g. build) and cd into that directory. ``` # ~/Development/code/apache/nifi-minifi-cpp on git:master @@ -403,6 +405,8 @@ advanced features. [100%] Built target run-tests ``` +#### Create MiNiFi Package using Standalone CMake + - Create a binary assembly located in your build directory with suffix .tar.gz ``` ~/Development/code/apache/nifi-minifi-cpp/build @@ -427,6 +431,11 @@ advanced features. CPack: - package: ~/Development/code/apache/nifi-minifi-cpp/build/nifi-minifi-cpp-0.99.0-source.tar.gz generated. ``` +#### Build MiNiFi & Create MiNiFi Package using Conan v2 + +Building MiNiFi and creating MiNiFi package supporting a portion of the extensions has been tested with Conan version 2 using VS code as an alternative to standalone CMake. By building MiNiFi using prebuilt conan packages for the external libraries as an alternative to CMake building the sources of those external libraries, we maybe able to speed up MiNiFi builds. Additionally, by creating a MiNiFi package as a conan package, it should be easier to integrate MiNiFi library and its executables into other C++ +project infrastructure to build out data pipelines on the edge. For instance, once we create the MiNiFi conan package, we can upload it to jfrog, conancenter or some other supported conan cloud repository and then download the prebuilt MiNiFi conan package to our new C++ project by adding it to our conanfile. For more details on the conan commands to build MiNiFi and create a MiNiFi conan package, see [CONAN.md](CONAN.md). + ### Building a docker image #### Building your own custom image From 7de03228f14050c4448104edff65a8be11cd97d0 Mon Sep 17 00:00:00 2001 From: james94 Date: Tue, 27 Aug 2024 16:38:29 -0700 Subject: [PATCH 19/26] Fixed the LoggerTests fmt formatting issue For conan, we install fmt 10.2.1 prebuilt conan package to not conflict with spdlog 1.14.0 prebuilt conan package. However, when we switched to a newer version of fmt, that caused the LoggerTest to fail, which didn't happen when we were using fmt 10.1.0 that was version installed when we used the standalone CMake approach. So, we updated our TEST_CASE for 'fmt formatting works with the logger' to explicity create the chrono minutes duration, followed by fmt format and then we pass our formatted minutes duration to our logger->log_critical, so we get the expected fmt formatting to be '1m'. We didn't have to do this explicitly when we used fmt 10.1.0 since even if we passed our logger->log_critical '1min', it would auto format to '1m'. For fmt 10.2.1 to have backward compatibility with fmt 10.1.0 and the result we expected from this TEST_CASE in LoggerTests, we made this update. I checked that when I build MiNiFi using conan and then build MiNiFi using standalone CMake, their LoggerTest PASSES. --- libminifi/test/unit/LoggerTests.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index 573ad20296..fc87ee5002 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -380,7 +380,9 @@ TEST_CASE("Setting max log entry length to unlimited results in unlimited log en TEST_CASE("fmt formatting works with the logger") { LogTestController::getInstance().setTrace(); std::shared_ptr logger = logging::LoggerFactory::getLogger(); - logger->log_critical("{} equals to {}", 1min, 60s); + std::chrono::minutes duration{1}; + std::string formatted_min_duration = fmt::format("{0}m", duration.count()); + logger->log_critical("{} equals to {}", formatted_min_duration, 60s); logger->log_critical("{} in hex is {:#x}", 13, 13); logger->log_critical("Unix epoch: {}", std::chrono::system_clock::time_point()); logger->log_critical("{:%Q %q} equals to {:%Q %q}", 2h, std::chrono::duration_cast(2h)); From 5f8ab375b39a2b22de51af8d4c99137f1324f38f Mon Sep 17 00:00:00 2001 From: james94 Date: Thu, 29 Aug 2024 14:11:31 -0700 Subject: [PATCH 20/26] Added zlib 1.2.11 conan package to resolve missing headers issue Now when we build MiNiFi using conan and run ctests, we shouldn't see that zlib missing headers issue --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 94bac3eebd..2d12c8e568 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,7 +7,7 @@ required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4") +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") From 742fe379f868e111681390efef5dab5420881f0b Mon Sep 17 00:00:00 2001 From: james94 Date: Mon, 2 Sep 2024 10:55:42 -0700 Subject: [PATCH 21/26] Updated conan to install zstd/1.5.2 & zlib/1.2.11 to successfully build BundledRocksDB On a fresh dev Ubuntu 22.04 docker environment that mainly comes with the apt packages needed to build MiNiFi, I realized that there were problems with compilation coming from BundledRocksDB, so once I updated the BundledRocksDB to temporarily account for conan packages zstd & zlib, BundledRocksDB built rocksdb system library. When building MiNiFi with conan, I later plan to switch from installing system library rocksdb to installing the conan package rocksdb with the needed patches. We were able to successfully install rocksdb sysem library when building MiNiFi with conan build and MiNiFi then built successfully using conan build. However, two ctests failed '34 - ProvenanceTests (Subprocess aborted)' and '219 - ControllerTests (Failed)'. I didn't see the compilation failure for MiNiFi conan build related to 'librdkafka/kafka-external-prefix/src/kafka-external/src/rdcrc32.h:57:10' for 'fatal error: zlib.h: No such file or directory'. --- cmake/BundledRocksDB.cmake | 20 ++++++++++++++++---- cmake/GetZstd.cmake | 27 +++++++++++++++++++++++++++ cmake/MiNiFiOptions.cmake | 1 + conanfile.py | 3 ++- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 cmake/GetZstd.cmake diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake index 2d4211e944..34882f26de 100644 --- a/cmake/BundledRocksDB.cmake +++ b/cmake/BundledRocksDB.cmake @@ -19,7 +19,8 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) message("Using bundled RocksDB") if (NOT WIN32) - include(Zstd) + include(GetZstd) + get_zstd() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy") include(LZ4) @@ -63,10 +64,17 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) -DROCKSDB_INSTALL_ON_WINDOWS=ON -DWITH_XPRESS=ON) else() + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=OFF + -DWITH_ZSTD=OFF) + else() + list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=ON + -DWITH_ZSTD=ON) + endif() list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=ON -DWITH_BZ2=ON - -DWITH_ZSTD=ON -DWITH_LZ4=ON) endif() @@ -95,7 +103,11 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) add_library(RocksDB::RocksDB STATIC IMPORTED) set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}") if (NOT WIN32) - add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4) + else() + add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) + endif() endif() add_dependencies(RocksDB::RocksDB rocksdb-external) file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR}) diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake new file mode 100644 index 0000000000..2b23929531 --- /dev/null +++ b/cmake/GetZstd.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_zstd) + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt zstd external lib") + find_package(zstd REQUIRED) + add_library(zstd::zstd ALIAS zstd::libzstd_static) + elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib") + include(Zstd) + endif() +endfunction(get_zstd) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index ae4a9f485a..13545bcfb0 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -138,6 +138,7 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") diff --git a/conanfile.py b/conanfile.py index 2d12c8e568..0f4f82b27b 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,7 +7,7 @@ required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11") +shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") @@ -31,6 +31,7 @@ def generate(self): tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN" tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" From 2f86d4bfbc39f8abed948f79a4d8f898a130905e Mon Sep 17 00:00:00 2001 From: james94 Date: Wed, 2 Oct 2024 00:15:40 -0700 Subject: [PATCH 22/26] Created RocksDB Conan Recipe to take patches like BundledRocksDB Also when we create the RocksDB conan package, it not only builds with the patches like BundledRocksDB, it also incorporates ZLib and ZStd conan packages too. Therefore, now that we use a conan RocksDB for building MiNiFi with conan, when we build MiNiFi with standalone CMake, we use the original version of BundledRocksDB that was in 'main' branch before I modified it. I verified that when I build MiNiFi with conan, 218 CTESTs pass out of 219. Similarly, when I build MiNiFi with standalone, then I run CTESTs, 218 out of 219 pass too. --- CMakeLists.txt | 6 +- CONAN.md | 13 + cmake/BundledRocksDB.cmake | 26 +- cmake/GetBZip2.cmake | 27 ++ cmake/GetCatch2.cmake | 2 +- cmake/GetCivetWeb.cmake | 2 +- cmake/GetFmt.cmake | 2 +- cmake/GetLibCURL.cmake | 2 +- cmake/GetLibXml2.cmake | 2 +- cmake/GetOpenSSL.cmake | 2 +- cmake/GetRocksDB.cmake | 34 +++ cmake/GetSpdlog.cmake | 2 +- cmake/GetZLIB.cmake | 2 +- cmake/GetZstd.cmake | 2 +- cmake/MiNiFiOptions.cmake | 2 + conanfile.py | 4 +- extensions/rocksdb-repos/CMakeLists.txt | 9 +- thirdparty/rocksdb/all/conandata.yml | 14 + thirdparty/rocksdb/all/conanfile.py | 243 ++++++++++++++++++ .../rocksdb/{ => all/patches}/arm7.patch | 2 +- .../rocksdb/{ => all/patches}/cstdint.patch | 6 +- .../dboptions_equality_operator.patch | 8 +- thirdparty/rocksdb/config.yml | 3 + 23 files changed, 369 insertions(+), 46 deletions(-) create mode 100644 cmake/GetBZip2.cmake create mode 100644 cmake/GetRocksDB.cmake create mode 100644 thirdparty/rocksdb/all/conandata.yml create mode 100644 thirdparty/rocksdb/all/conanfile.py rename thirdparty/rocksdb/{ => all/patches}/arm7.patch (91%) rename thirdparty/rocksdb/{ => all/patches}/cstdint.patch (97%) rename thirdparty/rocksdb/{ => all/patches}/dboptions_equality_operator.patch (98%) create mode 100644 thirdparty/rocksdb/config.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c460aa0fd..21aa51f223 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,8 +228,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) - include(BundledBZip2) - use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + include(GetBZip2) + get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + # include(BundledBZip2) + # use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy") endif() diff --git a/CONAN.md b/CONAN.md index adaf10d02d..088f343156 100644 --- a/CONAN.md +++ b/CONAN.md @@ -25,6 +25,7 @@ We will walk through the steps to build MiNiFi using conan version 2 that comes - [Build MiNiFi C++ with Conan](#build-minifi-c---with-conan) - [Create MiNiFi C++ Conan Package](#create-minifi-c---conan-package) - [Conclusion](#conclusion) + - [Appendix](#appendix) ## Description @@ -120,3 +121,15 @@ To have a more consistent quick build process for MiNiFi, we can use conan versi There are multiple benefits of having MiNiFi prebuilt conan packages. We can upload these MiNiFi conan packages to a conan repository like jfrog for version management. We can easily integrate MiNiFi's edge data pipeline features into other C++ software infrastructure using conan's CMake support. We can still use MiNiFi for edge data collection from the IoT devices embedded on robotic systems. We can integrate MiNiFi into self-driving cars (sensor examples: cameras, lidar, radar, inertial measurement unit (IMU), electronic speed controller (ESC), steering servo, etc), into medical imaging robots (sensor examples: depth cameras, ultrasound, gamma detector, force/torque sensor, joint position sensor, etc) or some other real-time robotic system. By leveraging MiNiFi as a conan package, we can leverage MiNiFi that comes with the best practices of building data pipelines from NiFi and bring them into existing C++ real-time robotics infrastructure. Some teams across companies typically have their own custom edge data pipelines that process data for the different events to eventually perform actions on that data. As an alternative to all these companies and their teams having their own custom edge data pipeline libraries, MiNiFi C++, which is like a headless NiFi, can provide a more consistent standard approach for team's to build edge pipelines. Through all stages of the edge data pipelines, MiNiFi can still provide telemetry to NiFi instances running in the cloud. + +## Appendix + +### Create Custom RocksDB Conan Package + +~~~bash +pushd nifi-minifi-cpp/thirdparty/rocksdb/all + +conan create . --user=minifi --channel=develop --version=8.10.2 -pr=../../../etc/conan/profiles/release-linux + +popd +~~~ diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake index 34882f26de..ac2a6e203a 100644 --- a/cmake/BundledRocksDB.cmake +++ b/cmake/BundledRocksDB.cmake @@ -19,8 +19,7 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) message("Using bundled RocksDB") if (NOT WIN32) - include(GetZstd) - get_zstd() + include(Zstd) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy") include(LZ4) @@ -28,9 +27,9 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) endif() # Patch to fix build issue on ARM7 architecture: https://github.com/facebook/rocksdb/issues/8609#issuecomment-1009572506 - set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/arm7.patch") - set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/dboptions_equality_operator.patch") - set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/cstdint.patch") + set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/arm7.patch") + set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch") + set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/cstdint.patch") set(PC ${Bash_EXECUTABLE} -c "set -x &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\") &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_2}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_2}\") &&\ @@ -64,17 +63,10 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) -DROCKSDB_INSTALL_ON_WINDOWS=ON -DWITH_XPRESS=ON) else() - if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=OFF - -DWITH_ZSTD=OFF) - else() - list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=ON - -DWITH_ZSTD=ON) - endif() list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=ON -DWITH_BZ2=ON + -DWITH_ZSTD=ON -DWITH_LZ4=ON) endif() @@ -103,11 +95,7 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) add_library(RocksDB::RocksDB STATIC IMPORTED) set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}") if (NOT WIN32) - if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4) - else() - add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) - endif() + add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) endif() add_dependencies(RocksDB::RocksDB rocksdb-external) file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR}) diff --git a/cmake/GetBZip2.cmake b/cmake/GetBZip2.cmake new file mode 100644 index 0000000000..be84ec1313 --- /dev/null +++ b/cmake/GetBZip2.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_bzip2 SOURCE_DIR BINARY_DIR) + if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt bzip2 external lib") + find_package(BZip2 REQUIRED) + elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD") + message("Using CMake to build bzip2 from source") + include(BundledBZip2) + use_bundled_bzip2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_bzip2) diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index 23ff9dbee7..b9d652a4ee 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -21,7 +21,7 @@ function(get_catch2) find_package(Catch2 REQUIRED) add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib") + message("Using CMake to build Catch2 from source") include(Catch2) endif() endfunction(get_catch2) diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake index 3e03b4fbc5..0657e03836 100644 --- a/cmake/GetCivetWeb.cmake +++ b/cmake/GetCivetWeb.cmake @@ -20,7 +20,7 @@ function(get_civetweb) message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") find_package(civetweb REQUIRED) elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib") + message("Using CMake to build CivetWeb from source") include(CivetWeb) endif() endfunction(get_civetweb) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index 94bce3c218..25b15aaf1b 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -20,7 +20,7 @@ function(get_fmt) message("Using Conan Packager to manage installing prebuilt Fmt external lib") find_package(fmt REQUIRED) elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib") + message("Using CMake to build Fmt from source") include(fmt) endif() endfunction(get_fmt) diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index b2dff1892c..86f8a87f75 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -20,7 +20,7 @@ function(get_curl SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt libcurl external lib") find_package(CURL REQUIRED) elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") + message("Using CMake to build libcurl from source") include(BundledLibcURL) use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index 369f1285c9..4e780e29c8 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -20,7 +20,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib") + message("Using CMake to build LibXml2 from source") include(BundledLibXml2) use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index 21d34dadb9..3029d7e1f0 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -20,7 +20,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") find_package(OpenSSL REQUIRED) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") + message("Using CMake to build OpenSSL from source") include(BundledOpenSSL) use_openssl(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetRocksDB.cmake b/cmake/GetRocksDB.cmake new file mode 100644 index 0000000000..095ae97386 --- /dev/null +++ b/cmake/GetRocksDB.cmake @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_rocksdb SOURCE_DIR BINARY_DIR) + if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt RocksDB external lib") + find_package(RocksDB REQUIRED) + add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb) + elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD") + message("Using CMake to build RocksDB from source") + + if (BUILD_ROCKSDB) + include(BundledRocksDB) + use_bundled_rocksdb(${SOURCE_DIR} ${BINARY_DIR}) + else() + list(APPEND CMAKE_MODULE_PATH "${SOURCE_DIR}/cmake/rocksdb/sys") + find_package(RocksDB REQUIRED) + endif() + endif() +endfunction(get_rocksdb SOURCE_DIR BINARY_DIR) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index b9f4233eff..b8b05f9992 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -25,7 +25,7 @@ function(get_spdlog) add_library(spdlog ALIAS spdlog::spdlog) elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") + message("Using CMake to build Spdlog from source") include(Spdlog) endif() endfunction(get_spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index 9b018df76e..8111ca631a 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -20,7 +20,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt zlib external lib") find_package(ZLIB REQUIRED) elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") + message("Using CMake to build zlib from source") include(BundledZLIB) use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake index 2b23929531..596df4aaef 100644 --- a/cmake/GetZstd.cmake +++ b/cmake/GetZstd.cmake @@ -21,7 +21,7 @@ function(get_zstd) find_package(zstd REQUIRED) add_library(zstd::zstd ALIAS zstd::libzstd_static) elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib") + message("Using CMake to build zstd from source") include(Zstd) endif() endfunction(get_zstd) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 13545bcfb0..8a7ccc58cf 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -138,7 +138,9 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ROCKSDB_SOURCE "Retrieves RocksDB from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_BZIP2_SOURCE "Retrieves BZip2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") diff --git a/conanfile.py b/conanfile.py index 0f4f82b27b..5dd4db1fca 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,7 +7,7 @@ required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2") +shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2", "bzip2/1.0.8", "rocksdb/8.10.2@minifi/develop") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") @@ -31,7 +31,9 @@ def generate(self): tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ROCKSDB_SOURCE"] = "CONAN" tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN" + tc.variables["MINIFI_BZIP2_SOURCE"] = "CONAN" tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt index e9d0a7af8d..2f43753c7e 100644 --- a/extensions/rocksdb-repos/CMakeLists.txt +++ b/extensions/rocksdb-repos/CMakeLists.txt @@ -21,13 +21,8 @@ if (NOT ENABLE_ROCKSDB) return() endif() -if (BUILD_ROCKSDB) - include(BundledRocksDB) - use_bundled_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) -else() - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/rocksdb/sys") - find_package(RocksDB REQUIRED) -endif() +include(GetRocksDB) +get_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) diff --git a/thirdparty/rocksdb/all/conandata.yml b/thirdparty/rocksdb/all/conandata.yml new file mode 100644 index 0000000000..ed09c733cf --- /dev/null +++ b/thirdparty/rocksdb/all/conandata.yml @@ -0,0 +1,14 @@ +sources: + "8.10.2": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.10.2.tar.gz" + sha256: "44b6ec2f4723a0d495762da245d4a59d38704e0d9d3d31c45af4014bee853256" +patches: + "8.10.2": + - patch_file: "patches/arm7.patch" + patch_description: "Fix for ARM7" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/36244376bb46c5c85657c6dfe92a6a0c4cea650a" + - patch_file: "patches/dboptions_equality_operator.patch" + patch_description: "Fix for missing equality operator in DBOptions and make DBOptions configurable" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/545236fd06b613f2d2b38f74ff9ee85df9190d59" diff --git a/thirdparty/rocksdb/all/conanfile.py b/thirdparty/rocksdb/all/conanfile.py new file mode 100644 index 0000000000..d9a5ae5627 --- /dev/null +++ b/thirdparty/rocksdb/all/conanfile.py @@ -0,0 +1,243 @@ +import os +import glob +import shutil + +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version + +required_conan_version = ">=1.53.0" + + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb" + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index" + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_bz2": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": True, + "with_zlib": True, + "with_bz2": True, + "with_zstd": True, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": True, + } + + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "20" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_bz2: + self.requires("bzip2/1.0.8") + # self.requires("bzip2/1.0.8@minifi/dev") # prebuilt with minifi bz2 patch + if self.options.with_zstd: + self.requires("zstd/1.5.2") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = True + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_BZ2"] = self.options.with_bz2 + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + if self.options.with_zstd: + deps.set_property("zstd", "cmake_target_name", "zstd::zstd") + deps.generate() + + def build(self): + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def package(self): + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) + if self.settings.os == "Windows": + self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] + if self.options.shared: + self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] + elif self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] + if self.options.lite: + self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") + if self.options.with_gflags: + self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") + if self.options.with_snappy: + self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") + if self.options.with_zlib: + self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") + if self.options.with_bz2: + self.cpp_info.components["librocksdb"].requires.append("bzip2::bzip2") + if self.options.with_zstd: + self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/thirdparty/rocksdb/arm7.patch b/thirdparty/rocksdb/all/patches/arm7.patch similarity index 91% rename from thirdparty/rocksdb/arm7.patch rename to thirdparty/rocksdb/all/patches/arm7.patch index 1eb64ed5ce..bd1eb6aa22 100644 --- a/thirdparty/rocksdb/arm7.patch +++ b/thirdparty/rocksdb/all/patches/arm7.patch @@ -4,7 +4,7 @@ index 225e3fa72..cd5f935f1 100644 +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h @@ -131,6 +131,10 @@ static inline tokutime_t toku_time_now(void) { uint64_t result; - __asm __volatile__("mrs %[rt], cntvct_el0" : [ rt ] "=r"(result)); + __asm __volatile__("mrs %[rt], cntvct_el0" : [rt] "=r"(result)); return result; +#elif defined(__arm__) + uint32_t lo, hi; diff --git a/thirdparty/rocksdb/cstdint.patch b/thirdparty/rocksdb/all/patches/cstdint.patch similarity index 97% rename from thirdparty/rocksdb/cstdint.patch rename to thirdparty/rocksdb/all/patches/cstdint.patch index 2435905c9a..4cff815831 100644 --- a/thirdparty/rocksdb/cstdint.patch +++ b/thirdparty/rocksdb/all/patches/cstdint.patch @@ -3,10 +3,10 @@ index 75d61abb4..f42ef6dc2 100644 --- a/options/offpeak_time_info.h +++ b/options/offpeak_time_info.h @@ -5,6 +5,7 @@ - + #pragma once - + +#include #include - + #include "rocksdb/rocksdb_namespace.h" diff --git a/thirdparty/rocksdb/dboptions_equality_operator.patch b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch similarity index 98% rename from thirdparty/rocksdb/dboptions_equality_operator.patch rename to thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch index 2efb91ea0e..cff3dd31ea 100644 --- a/thirdparty/rocksdb/dboptions_equality_operator.patch +++ b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch @@ -3,19 +3,19 @@ index ae5ed2c26..0038c6bff 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -397,6 +397,7 @@ struct DbPath { - + DbPath() : target_size(0) {} DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {} + bool operator==(const DbPath& other) const = default; }; - + extern const char* kHostnameForDbHostId; @@ -1008,6 +1009,8 @@ struct DBOptions { // Create DBOptions from Options explicit DBOptions(const Options& options); - + + bool operator==(const DBOptions& other) const = default; + void Dump(Logger* log) const; - + // Allows OS to incrementally sync files to disk while they are being diff --git a/thirdparty/rocksdb/config.yml b/thirdparty/rocksdb/config.yml new file mode 100644 index 0000000000..21a3c25eab --- /dev/null +++ b/thirdparty/rocksdb/config.yml @@ -0,0 +1,3 @@ +versions: + "8.10.2": + folder: all From 444bcd3ea6c5608e132c6013f304dc53a38d6249 Mon Sep 17 00:00:00 2001 From: james94 Date: Thu, 3 Oct 2024 00:11:06 -0700 Subject: [PATCH 23/26] Updated conan profile to ISO CPP & Get.cmake conan msg to be concise Addressed Marton's feedback for conan linux profile to use ISO C++ & Then updated the Get.cmake files messages related to conan to be more concise. Rebuilt MiNiFi with conan and then standalone cmake. Also created MiNiFi conan package. In these 3 cases, I also checked the CTESTs to see that 218 out of 219 TEST CASES passed. '219 - ControllerTests (Failed)' and thats expected for now since LIBARCHIVE is DISABLED. --- cmake/GetBZip2.cmake | 2 +- cmake/GetCatch2.cmake | 2 +- cmake/GetCivetWeb.cmake | 2 +- cmake/GetFmt.cmake | 2 +- cmake/GetLibCURL.cmake | 2 +- cmake/GetLibXml2.cmake | 2 +- cmake/GetOpenSSL.cmake | 2 +- cmake/GetRocksDB.cmake | 2 +- cmake/GetSpdlog.cmake | 2 +- cmake/GetZLIB.cmake | 2 +- cmake/GetZstd.cmake | 2 +- etc/conan/profiles/release-linux | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmake/GetBZip2.cmake b/cmake/GetBZip2.cmake index be84ec1313..9c886a776f 100644 --- a/cmake/GetBZip2.cmake +++ b/cmake/GetBZip2.cmake @@ -17,7 +17,7 @@ function(get_bzip2 SOURCE_DIR BINARY_DIR) if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt bzip2 external lib") + message("Using Conan Packager to install bzip2") find_package(BZip2 REQUIRED) elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD") message("Using CMake to build bzip2 from source") diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index b9d652a4ee..4c2a48e5e1 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -17,7 +17,7 @@ function(get_catch2) if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt Catch2 external lib") + message("Using Conan Packager to install Catch2") find_package(Catch2 REQUIRED) add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake index 0657e03836..09dc510c8b 100644 --- a/cmake/GetCivetWeb.cmake +++ b/cmake/GetCivetWeb.cmake @@ -17,7 +17,7 @@ function(get_civetweb) if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") + message("Using Conan Packager to install CivetWeb") find_package(civetweb REQUIRED) elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") message("Using CMake to build CivetWeb from source") diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index 25b15aaf1b..54c7adbaba 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -17,7 +17,7 @@ function(get_fmt) if(MINIFI_FMT_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt Fmt external lib") + message("Using Conan Packager to install Fmt") find_package(fmt REQUIRED) elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") message("Using CMake to build Fmt from source") diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index 86f8a87f75..baa07cb6b4 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -17,7 +17,7 @@ function(get_curl SOURCE_DIR BINARY_DIR) if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt libcurl external lib") + message("Using Conan Packager to install libcurl") find_package(CURL REQUIRED) elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") message("Using CMake to build libcurl from source") diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index 4e780e29c8..d8d5e22c68 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -17,7 +17,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") + message("Using Conan Packager to install LibXml2") find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") message("Using CMake to build LibXml2 from source") diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index 3029d7e1f0..cff364b9ab 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -17,7 +17,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") + message("Using Conan Packager to install OpenSSL") find_package(OpenSSL REQUIRED) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") message("Using CMake to build OpenSSL from source") diff --git a/cmake/GetRocksDB.cmake b/cmake/GetRocksDB.cmake index 095ae97386..eb555b7ce9 100644 --- a/cmake/GetRocksDB.cmake +++ b/cmake/GetRocksDB.cmake @@ -17,7 +17,7 @@ function(get_rocksdb SOURCE_DIR BINARY_DIR) if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt RocksDB external lib") + message("Using Conan Packager to install RocksDB") find_package(RocksDB REQUIRED) add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb) elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD") diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index b8b05f9992..f342e4be83 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -20,7 +20,7 @@ function(get_spdlog) get_fmt() if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt Spdlog external lib") + message("Using Conan Packager to install Spdlog") find_package(spdlog REQUIRED) add_library(spdlog ALIAS spdlog::spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index 8111ca631a..5a66f37685 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -17,7 +17,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR) if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt zlib external lib") + message("Using Conan Packager to install zlib") find_package(ZLIB REQUIRED) elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") message("Using CMake to build zlib from source") diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake index 596df4aaef..ebf2d62941 100644 --- a/cmake/GetZstd.cmake +++ b/cmake/GetZstd.cmake @@ -17,7 +17,7 @@ function(get_zstd) if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to manage installing prebuilt zstd external lib") + message("Using Conan Packager to install zstd") find_package(zstd REQUIRED) add_library(zstd::zstd ALIAS zstd::libzstd_static) elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") diff --git a/etc/conan/profiles/release-linux b/etc/conan/profiles/release-linux index 26b4fa6a08..9233901a5f 100644 --- a/etc/conan/profiles/release-linux +++ b/etc/conan/profiles/release-linux @@ -3,7 +3,7 @@ os=Linux arch=x86_64 build_type=Release compiler=gcc -compiler.cppstd=gnu20 +compiler.cppstd=20 compiler.libcxx=libstdc++11 compiler.version=11 [conf] From d5e672bb10e23ffeef131eba014cfd715a65c3f5 Mon Sep 17 00:00:00 2001 From: james94 Date: Thu, 3 Oct 2024 09:33:25 -0700 Subject: [PATCH 24/26] Addressed Martons Feedback Reverted rocksdb patches: cstdint.patch and doptions_equality_operator.patch by copying over the ones from MiNiFi main branch. Kept the brief update I made to arm7.patch since when I tried the original version of arm7.patch from main branch, conan complained when trying to apply the patch. Added the warning message in CONAN.md about conan integration with MiNiFi being experimental. Updated the messages for GetLibXml2.cmake and GetSpdlog.cmake. Verified I can build MiNiFi with standalone CMake, conan build and conan create --- CMakeLists.txt | 2 -- CONAN.md | 2 ++ cmake/GetLibXml2.cmake | 4 ++-- cmake/GetSpdlog.cmake | 4 ++-- thirdparty/rocksdb/all/patches/cstdint.patch | 6 +++--- .../rocksdb/all/patches/dboptions_equality_operator.patch | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21aa51f223..9b31d6c0e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,8 +230,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) include(GetBZip2) get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) - # include(BundledBZip2) - # use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy") endif() diff --git a/CONAN.md b/CONAN.md index 088f343156..f4a5d04f20 100644 --- a/CONAN.md +++ b/CONAN.md @@ -17,6 +17,8 @@ We will walk through the steps to build MiNiFi using conan version 2 that comes with CMake integration. We will also go through the process of creating a MiNiFi conan package for easier integration into other C++ projects. We will start with a discussion on building MiNiFi using the standalone CMake approach and the conan approach. After we have built MiNiFi and created a MiNiFi conan package, we will conclude by elaborating on some of the benefits that can come with integrating a MiNiFi conan package into new or existing C++ infrastructure with respect to real-time robotic systems. +Conan build support is experimental, and not yet suitable for general use. + ## Table of Contents - [Apache NiFi - MiNiFi - C++ Conan Build Guide](#apache-nifi---minifi---c---conan-build-guide) diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index d8d5e22c68..4e3924ef64 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -17,10 +17,10 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install LibXml2") + message("Using Conan Packager to install libxml2") find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") - message("Using CMake to build LibXml2 from source") + message("Using CMake to build libxml2 from source") include(BundledLibXml2) use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index f342e4be83..acffd7bb1e 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -20,12 +20,12 @@ function(get_spdlog) get_fmt() if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install Spdlog") + message("Using Conan Packager to install spdlog") find_package(spdlog REQUIRED) add_library(spdlog ALIAS spdlog::spdlog) elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") - message("Using CMake to build Spdlog from source") + message("Using CMake to build spdlog from source") include(Spdlog) endif() endfunction(get_spdlog) diff --git a/thirdparty/rocksdb/all/patches/cstdint.patch b/thirdparty/rocksdb/all/patches/cstdint.patch index 4cff815831..2435905c9a 100644 --- a/thirdparty/rocksdb/all/patches/cstdint.patch +++ b/thirdparty/rocksdb/all/patches/cstdint.patch @@ -3,10 +3,10 @@ index 75d61abb4..f42ef6dc2 100644 --- a/options/offpeak_time_info.h +++ b/options/offpeak_time_info.h @@ -5,6 +5,7 @@ - + #pragma once - + +#include #include - + #include "rocksdb/rocksdb_namespace.h" diff --git a/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch index cff3dd31ea..2efb91ea0e 100644 --- a/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch +++ b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch @@ -3,19 +3,19 @@ index ae5ed2c26..0038c6bff 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -397,6 +397,7 @@ struct DbPath { - + DbPath() : target_size(0) {} DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {} + bool operator==(const DbPath& other) const = default; }; - + extern const char* kHostnameForDbHostId; @@ -1008,6 +1009,8 @@ struct DBOptions { // Create DBOptions from Options explicit DBOptions(const Options& options); - + + bool operator==(const DBOptions& other) const = default; + void Dump(Logger* log) const; - + // Allows OS to incrementally sync files to disk while they are being From 588277f66edc3b0bb4dc9cb1a51139f1514a46f4 Mon Sep 17 00:00:00 2001 From: james94 Date: Fri, 4 Oct 2024 00:06:54 -0700 Subject: [PATCH 25/26] Reverted RocksDB arm7.patch based on main branch one & created conan package After reverting RocksDB arm7.patch back to the one based on MiNiFi main branch, verified I could still create custom RocksDB conan package using Fuzzy patching. Fuzzy patching allows us to proceed creating a conan package like RocksDB where even if there is a minor difference between patch code and actual source code, we can still apply the patch. This fuzzy patching was key in us being able to successfully apply arm7.patch when creating a custom RocksDB conan package. Previously we used apply_conandata_patches() when creating the custom RocksDB conan package and it was too strict, so we'll stick with the fuzzy patching approach. Then I verified I could build MiNiFi using the conan build approach and then the standalone CMake approach. Afterward, I verified I could create the MiNiFi conan package. Finally, I double checked that 218 out of 219 CTEST cases passed for building MiNiFi using conan build and then standalone CMake approaches. 218 out of 219 CTEST cases passed after creating the MiNiFi conan package. --- thirdparty/rocksdb/all/conandata.yml | 1 + thirdparty/rocksdb/all/conanfile.py | 6 +++--- thirdparty/rocksdb/all/patches/arm7.patch | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/thirdparty/rocksdb/all/conandata.yml b/thirdparty/rocksdb/all/conandata.yml index ed09c733cf..f5c3ecd302 100644 --- a/thirdparty/rocksdb/all/conandata.yml +++ b/thirdparty/rocksdb/all/conandata.yml @@ -12,3 +12,4 @@ patches: patch_description: "Fix for missing equality operator in DBOptions and make DBOptions configurable" patch_type: "portability" patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/545236fd06b613f2d2b38f74ff9ee85df9190d59" + - patch_file: "patches/cstdint.patch" diff --git a/thirdparty/rocksdb/all/conanfile.py b/thirdparty/rocksdb/all/conanfile.py index d9a5ae5627..a1ff97ad87 100644 --- a/thirdparty/rocksdb/all/conanfile.py +++ b/thirdparty/rocksdb/all/conanfile.py @@ -6,7 +6,7 @@ from conan.errors import ConanInvalidConfiguration from conan.tools.build import check_min_cppstd from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout -from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.files import patch, collect_libs, copy, export_conandata_patches, get, rm, rmdir from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime from conan.tools.scm import Version @@ -96,7 +96,6 @@ def requirements(self): self.requires("zlib/[>=1.2.11 <2]") if self.options.with_bz2: self.requires("bzip2/1.0.8") - # self.requires("bzip2/1.0.8@minifi/dev") # prebuilt with minifi bz2 patch if self.options.with_zstd: self.requires("zstd/1.5.2") if self.options.get_safe("with_tbb"): @@ -174,7 +173,8 @@ def generate(self): deps.generate() def build(self): - apply_conandata_patches(self) + for patch_data in self.conan_data.get("patches", {}).get(self.version, []): + patch(self, patch_file=patch_data["patch_file"], base_path=self.source_folder, strip=1, fuzz=True) cmake = CMake(self) cmake.configure() cmake.build() diff --git a/thirdparty/rocksdb/all/patches/arm7.patch b/thirdparty/rocksdb/all/patches/arm7.patch index bd1eb6aa22..1eb64ed5ce 100644 --- a/thirdparty/rocksdb/all/patches/arm7.patch +++ b/thirdparty/rocksdb/all/patches/arm7.patch @@ -4,7 +4,7 @@ index 225e3fa72..cd5f935f1 100644 +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h @@ -131,6 +131,10 @@ static inline tokutime_t toku_time_now(void) { uint64_t result; - __asm __volatile__("mrs %[rt], cntvct_el0" : [rt] "=r"(result)); + __asm __volatile__("mrs %[rt], cntvct_el0" : [ rt ] "=r"(result)); return result; +#elif defined(__arm__) + uint32_t lo, hi; From ff747e1af1d7401faa52e11e071a9d29a1159677 Mon Sep 17 00:00:00 2001 From: james94 Date: Wed, 9 Oct 2024 00:35:15 -0700 Subject: [PATCH 26/26] Updated Conan Package to Just 'Conan' for Get.cmake messages Verified I can build MiNiFi with conan build and standalone cmake. Verified I can create MiNIFi conan package with conan create. --- cmake/GetBZip2.cmake | 2 +- cmake/GetCatch2.cmake | 2 +- cmake/GetCivetWeb.cmake | 2 +- cmake/GetFmt.cmake | 2 +- cmake/GetLibCURL.cmake | 2 +- cmake/GetLibXml2.cmake | 2 +- cmake/GetOpenSSL.cmake | 2 +- cmake/GetRocksDB.cmake | 2 +- cmake/GetSpdlog.cmake | 2 +- cmake/GetZLIB.cmake | 2 +- cmake/GetZstd.cmake | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmake/GetBZip2.cmake b/cmake/GetBZip2.cmake index 9c886a776f..2a7d47e9e7 100644 --- a/cmake/GetBZip2.cmake +++ b/cmake/GetBZip2.cmake @@ -17,7 +17,7 @@ function(get_bzip2 SOURCE_DIR BINARY_DIR) if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install bzip2") + message("Using Conan to install bzip2") find_package(BZip2 REQUIRED) elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD") message("Using CMake to build bzip2 from source") diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index 4c2a48e5e1..d6c1d6acca 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -17,7 +17,7 @@ function(get_catch2) if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install Catch2") + message("Using Conan to install Catch2") find_package(Catch2 REQUIRED) add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake index 09dc510c8b..f30d222897 100644 --- a/cmake/GetCivetWeb.cmake +++ b/cmake/GetCivetWeb.cmake @@ -17,7 +17,7 @@ function(get_civetweb) if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install CivetWeb") + message("Using Conan to install CivetWeb") find_package(civetweb REQUIRED) elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") message("Using CMake to build CivetWeb from source") diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index 54c7adbaba..f4f28458db 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -17,7 +17,7 @@ function(get_fmt) if(MINIFI_FMT_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install Fmt") + message("Using Conan to install Fmt") find_package(fmt REQUIRED) elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") message("Using CMake to build Fmt from source") diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index baa07cb6b4..afd38f3c9c 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -17,7 +17,7 @@ function(get_curl SOURCE_DIR BINARY_DIR) if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install libcurl") + message("Using Conan to install libcurl") find_package(CURL REQUIRED) elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") message("Using CMake to build libcurl from source") diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index 4e3924ef64..5915b117d6 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -17,7 +17,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install libxml2") + message("Using Conan to install libxml2") find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") message("Using CMake to build libxml2 from source") diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index cff364b9ab..58132c6bf4 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -17,7 +17,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install OpenSSL") + message("Using Conan to install OpenSSL") find_package(OpenSSL REQUIRED) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") message("Using CMake to build OpenSSL from source") diff --git a/cmake/GetRocksDB.cmake b/cmake/GetRocksDB.cmake index eb555b7ce9..0968afc381 100644 --- a/cmake/GetRocksDB.cmake +++ b/cmake/GetRocksDB.cmake @@ -17,7 +17,7 @@ function(get_rocksdb SOURCE_DIR BINARY_DIR) if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install RocksDB") + message("Using Conan to install RocksDB") find_package(RocksDB REQUIRED) add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb) elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD") diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index acffd7bb1e..747894984b 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -20,7 +20,7 @@ function(get_spdlog) get_fmt() if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install spdlog") + message("Using Conan to install spdlog") find_package(spdlog REQUIRED) add_library(spdlog ALIAS spdlog::spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index 5a66f37685..6e7bea1e33 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -17,7 +17,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR) if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install zlib") + message("Using Conan to install zlib") find_package(ZLIB REQUIRED) elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") message("Using CMake to build zlib from source") diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake index ebf2d62941..a0334b02de 100644 --- a/cmake/GetZstd.cmake +++ b/cmake/GetZstd.cmake @@ -17,7 +17,7 @@ function(get_zstd) if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN") - message("Using Conan Packager to install zstd") + message("Using Conan to install zstd") find_package(zstd REQUIRED) add_library(zstd::zstd ALIAS zstd::libzstd_static) elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD")