From 763026fafd620f08ce9186d2789735768c4c3ed0 Mon Sep 17 00:00:00 2001 From: Kirill Golikov Date: Mon, 30 Sep 2024 14:29:20 +0200 Subject: [PATCH] [NeoOnnx] Bump protobuf (#1115) * [NeoOnnx] Bump protobuf Signed-off-by: Kirill Golikov * [NeoOnnx] Update Sources.txt Signed-off-by: Kirill Golikov --------- Signed-off-by: Kirill Golikov --- Build/Sources.ThirdParty.txt | 13 ---- NeoML/Python/CMakeLists.txt | 14 ++-- NeoML/Python/setup.py | 2 +- NeoOnnx/src/CMakeLists.txt | 89 +++++++++++++----------- NeoOnnx/src/Operators/GatherOperator.cpp | 4 +- Sources.txt | 2 +- 6 files changed, 59 insertions(+), 65 deletions(-) delete mode 100644 Build/Sources.ThirdParty.txt diff --git a/Build/Sources.ThirdParty.txt b/Build/Sources.ThirdParty.txt deleted file mode 100644 index 38eb56067..000000000 --- a/Build/Sources.ThirdParty.txt +++ /dev/null @@ -1,13 +0,0 @@ - -git;https://tfs/HQ/ThirdParty/_git/CUDA -get;ThirdParty/CUDA;/;CUDA v11.2 1.0.2.0 - -git;https://tfs/HQ/ThirdParty/_git/Vulkan -get;ThirdParty/Vulkan;/;Vulkan SDK 1.1.130 1.0.2.0 - -git;https://tfs/HQ/ThirdParty/_git/MKL -get;ThirdParty/MKL;/;MKL 2022.0.2 1.0.6.0 - -git;https://tfs/HQ/ThirdParty/_git/protobuf -get;ThirdParty/protobuf;/;v3.11.4 - diff --git a/NeoML/Python/CMakeLists.txt b/NeoML/Python/CMakeLists.txt index 994a715fd..4662748a7 100644 --- a/NeoML/Python/CMakeLists.txt +++ b/NeoML/Python/CMakeLists.txt @@ -27,12 +27,16 @@ if(NOT pybind11_POPULATED) endif() # Build NeoML -set(NeoML_INSTALL OFF CACHE BOOL "" FORCE) -set(NeoML_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(NeoOnnx_BUILD ON CACHE BOOL "" FORCE) -if(LINUX) - set(NeoOnnx_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(NeoML_INSTALL OFF CACHE BOOL "" FORCE) +set(NeoML_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(NeoML_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(NeoOnnx_BUILD ON CACHE BOOL "" FORCE) +set(NeoOnnx_BUILD_SHARED OFF CACHE BOOL "" FORCE) + +if(MSVC) # static linking the standard library + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") endif() + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_BINARY_DIR}/NeoML) # Build python module diff --git a/NeoML/Python/setup.py b/NeoML/Python/setup.py index b7526c588..abaebd0ba 100644 --- a/NeoML/Python/setup.py +++ b/NeoML/Python/setup.py @@ -110,7 +110,7 @@ def get_long_description(): long_description=get_long_description(), long_description_content_type='text/markdown', url='http://github.com/neoml-lib/neoml', - install_requires=['numpy>=2.0.2', 'scipy>=1.5.2', 'onnx==1.16.0', 'protobuf==3.20.*'], + install_requires=['numpy>=2.0.2', 'scipy>=1.5.2', 'onnx==1.16.0', 'protobuf==5.28.*'], ext_modules=[CMakeExtension("neoml.PythonWrapper")], cmdclass={"build_ext": CMakeBuild}, include_package_data=True, diff --git a/NeoOnnx/src/CMakeLists.txt b/NeoOnnx/src/CMakeLists.txt index c2272ab16..283d55a28 100644 --- a/NeoOnnx/src/CMakeLists.txt +++ b/NeoOnnx/src/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.11 FATAL_ERROR) +set(CMAKE_CXX_STANDARD 14) + if(NeoOnnx_BUILD_SHARED) add_library(${PROJECT_NAME} SHARED common.cpp) else() @@ -142,62 +144,70 @@ endif() configure_target(${PROJECT_NAME}) +# Have to first set the compile flags and then go through your subfolders. +if(MSVC) + target_compile_options(${PROJECT_NAME} PRIVATE /W2) + if(NeoOnnx_BUILD_SHARED) + target_compile_options(${PROJECT_NAME} PRIVATE /MD$<$:d>) + else() + target_compile_options(${PROJECT_NAME} PRIVATE /MT$<$:d>) + endif() + # Because of the code generated by protobuf + set(CMAKE_CXX_FLAGS " /wd4946 /wd4251 /wd4141 /wd4068 /wd4530 /wd4715 /wd4018 /wd4267 /wd4267 ") +elseif(LINUX OR DARWIN) + target_compile_options(${PROJECT_NAME} PUBLIC -Wno-pedantic -Wno-array-bounds) + if(USE_FINE_OBJECTS) + target_compile_options(${PROJECT_NAME} PRIVATE -Wno-nonportable-include-path) + endif() + add_compile_options( -Wno-attributes ) +endif() + if(DEFINED ENV{Protobuf_ROOT} AND (IOS OR ANDROID)) list(APPEND CMAKE_FIND_ROOT_PATH $ENV{Protobuf_ROOT}) endif() +#---------------------------------------------------------------------------------------------------------------------- + +set(CMAKE_FOLDER "protobuf") + +#-------------------------- if(${CMAKE_VERSION} VERSION_GREATER "3.16.0") set(CMAKE_UNITY_BUILD OFF) endif() -set(protobuf_BUILD_TESTS OFF CACHE BOOL "") -set(protobuf_MSVC_STATIC_RUNTIME "${STATIC_NEOML}" CACHE BOOL "") -set(protobuf_WITH_ZLIB OFF CACHE BOOL "") -set(protobuf_BUILD_PROTOC_BINARIES ON CACHE BOOL "") -set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "") +set(protobuf_INSTALL OFF CACHE BOOL "Install protobuf binaries and files" ) +set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build tests" ) +set(protobuf_BUILD_LIBUPB OFF CACHE BOOL "Build libupb" ) +set(protobuf_BUILD_PROTOC_BINARIES ON CACHE BOOL "Build libprotoc and protoc compiler" ) +set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "Build Shared Libraries" ) +set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" ) +set(protobuf_MSVC_STATIC_RUNTIME ON CACHE BOOL "Link static runtime libraries" ) +set(ABSL_IDE_FOLDER "protobuf/absl" CACHE BOOL "The IDE sub-folder" ) if(USE_FINE_OBJECTS) - add_subdirectory("${FINE_ROOT}/ThirdParty/protobuf/cmake" + add_subdirectory("${FINE_ROOT}/ThirdParty/protobuf" "${CMAKE_BINARY_DIR}/protobuf" EXCLUDE_FROM_ALL) else() include(FetchContent) FetchContent_Declare( protobuf GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git - GIT_TAG v3.11.4 + GIT_TAG v28.0 ) FetchContent_GetProperties(protobuf) if(NOT protobuf_POPULATED) FetchContent_Populate(protobuf) - add_subdirectory(${protobuf_SOURCE_DIR}/cmake ${protobuf_BINARY_DIR} EXCLUDE_FROM_ALL) + add_subdirectory(${protobuf_SOURCE_DIR} ${protobuf_BINARY_DIR} EXCLUDE_FROM_ALL) endif() endif() -if(LINUX) - target_compile_options(libprotobuf PRIVATE -Wno-stringop-overflow) - target_compile_options(libprotobuf-lite PRIVATE -Wno-stringop-overflow) -endif() - -if(DARWIN) - target_compile_options(libprotobuf PRIVATE -Wno-deprecated-declarations) - target_compile_options(libprotobuf-lite PRIVATE -Wno-deprecated-declarations) -endif() - +#-------------------------- if(${CMAKE_VERSION} VERSION_GREATER "3.16.0") set(CMAKE_UNITY_BUILD ${NeoML_UNITY_BUILD}) endif() -if(USE_FINE_OBJECTS) - target_link_libraries(${PROJECT_NAME} PRIVATE FineObjects) - - # Todo: properly link FineStlStaticPart if any problems occure - if(MSVC) - target_link_libraries(${PROJECT_NAME} PRIVATE "msvcprt$<$:d>") - endif() -endif() - set(PROTO_DIR ${CMAKE_CURRENT_BINARY_DIR}/cpp_proto/${CMAKE_CFG_INTDIR}) set(PROTO_SRCS ${PROTO_DIR}/onnx.pb.cc) set(PROTO_HDRS ${PROTO_DIR}/onnx.pb.h) @@ -216,15 +226,15 @@ add_custom_command( target_sources(${PROJECT_NAME} PRIVATE ${PROTO_HDRS} ${PROTO_SRCS}) set_property(SOURCE ${PROTO_HDRS} ${PROTO_SRCS} PROPERTY SKIP_UNITY_BUILD_INCLUSION ON) -if(MSVC) - target_compile_options(${PROJECT_NAME} PRIVATE /W2) - target_compile_options(${PROJECT_NAME} PRIVATE /MD$<$:d>) - # Because of the code generated by protobuf - target_compile_options(${PROJECT_NAME} PRIVATE /wd4946 /wd4251) -elseif(NOT WIN32) - set_property(SOURCE ${PROTO_SRCS} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-pedantic -Wno-array-bounds ") - if(USE_FINE_OBJECTS) - target_compile_options(${PROJECT_NAME} PRIVATE -Wno-nonportable-include-path) +unset(CMAKE_FOLDER) + +#---------------------------------------------------------------------------------------------------------------------- + +if(USE_FINE_OBJECTS) + target_link_libraries(${PROJECT_NAME} PRIVATE FineObjects) + # TODO: properly link FineStlStaticPart if any problems occure + if(MSVC) + target_link_libraries(${PROJECT_NAME} PRIVATE "msvcprt$<$:d>") endif() endif() @@ -261,12 +271,7 @@ if(NOT TARGET protobuf::libprotobuf) message(FATAL_ERROR "Protobuf not found!") endif() -# Fix for a known protobuf Windows dll issue https://github.com/protocolbuffers/protobuf/issues/2502 -if((Protobuf_LIBRARY MATCHES ".*dll") OR (Protobuf_LIBRARY MATCHES ".*DLL")) - target_compile_definitions(${PROJECT_NAME} PRIVATE PROTOBUF_USE_DLLS) -endif() - -target_link_libraries(${PROJECT_NAME} PRIVATE protobuf::libprotobuf PUBLIC NeoML) +target_link_libraries(${PROJECT_NAME} PRIVATE protobuf::libprotobuf ${protobuf_ABSL_USED_TARGETS} PUBLIC NeoML) if(ANDROID) target_link_libraries(${PROJECT_NAME} PRIVATE log) endif() diff --git a/NeoOnnx/src/Operators/GatherOperator.cpp b/NeoOnnx/src/Operators/GatherOperator.cpp index 36c3508ed..67c75a2b7 100644 --- a/NeoOnnx/src/Operators/GatherOperator.cpp +++ b/NeoOnnx/src/Operators/GatherOperator.cpp @@ -16,8 +16,6 @@ limitations under the License. #include "../common.h" #pragma hdrstop -#include - #include "GatherOperator.h" #include "NeoOnnxCheck.h" @@ -57,7 +55,7 @@ void CGatherOperator::AddLayers( const CTensorArray& inputs, CDnn& dnn, CTensorA CPtr data = inputs[0]; CTensorLayout dataLayout; for( int i = 0; i < data->DimCount(); ++i ) { - dataLayout.Add( static_cast( std::max( 0, indices->DimCount() - 1 ) + i ) ); + dataLayout.Add( static_cast( max( 0, indices->DimCount() - 1 ) + i ) ); } if( axis != 0 ) { std::swap( dataLayout[0], dataLayout[axis] ); diff --git a/Sources.txt b/Sources.txt index 104ac58b8..995815420 100644 --- a/Sources.txt +++ b/Sources.txt @@ -12,7 +12,7 @@ git;https://github.com/abbyyProduct/ThirdParty-MKL get;ThirdParty/MKL;/;MKL 2022.0.2 1.0.8.0 git;https://github.com/abbyyProduct/ThirdParty-protobuf -get;ThirdParty/protobuf;/;v3.11.4 +get;ThirdParty/protobuf;/;protobuf-v28.0 git;https://github.com/abbyyProduct/Tools_Libs-NeoMLTest get;NeoMLTest;/;NeoMLTest-master 1.0.74.0