From 7fbdc31c9de27178db27aab95d40cb047c1ceb01 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 03:37:18 -0500 Subject: [PATCH 01/31] static gcc and libcurl --- components/core/src/clp_s/CMakeLists.txt | 20 ++++++++++++++++++++ components/core/src/clp_s/InputConfig.cpp | 5 +++++ components/core/src/clp_s/JsonParser.cpp | 2 ++ components/core/src/clp_s/clp-s.cpp | 2 ++ 4 files changed, 29 insertions(+) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index d38c28ade5..8a9e3f966a 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -1,3 +1,5 @@ +option(CLP_S_STATIC_EXE "Enable clp-s static exe" OFF) + add_subdirectory(indexer) add_subdirectory(log_converter) add_subdirectory(search) @@ -210,6 +212,11 @@ if(CLP_BUILD_CLP_S_IO) ${CLP_S_IO_SOURCES} ) add_library(clp_s::io ALIAS clp_s_io) + target_compile_definitions( + clp_s_io + PRIVATE + CLP_S_STATIC_EXE=$ + ) target_compile_features(clp_s_io PRIVATE cxx_std_20) target_include_directories(clp_s_io PUBLIC ../) target_link_libraries( @@ -428,6 +435,11 @@ if(CLP_BUILD_EXECUTABLES) clp-s.cpp ${CLP_S_EXE_SOURCES} ) + target_compile_definitions( + clp-s + PRIVATE + CLP_S_STATIC_EXE=$ + ) target_compile_features(clp-s PRIVATE cxx_std_20) target_link_libraries( clp-s @@ -452,4 +464,12 @@ if(CLP_BUILD_EXECUTABLES) PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}" ) + if(CLP_S_STATIC_EXE) + target_link_options( + clp-s + PRIVATE + -static-libgcc + -static-libstdc++ + ) + endif() endif() diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index d4c892a14f..d0d10cc1cb 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -193,6 +193,10 @@ auto try_sign_url(std::string& url) -> bool { auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) -> std::shared_ptr { +#if CLP_S_STATIC_EXE + SPDLOG_ERROR("Static clp-s executable currently does not support NetworkReader."); + return nullptr; +#else std::string request_url{url}; switch (auth.method) { case AuthMethod::S3PresignedUrlV4: @@ -212,6 +216,7 @@ auto try_create_network_reader(std::string_view const url, NetworkAuthOption con SPDLOG_ERROR("Failed to open url for reading - {}", e.what()); return nullptr; } +#endif } auto could_be_zstd(char const* peek_buf, size_t peek_size) -> bool { diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index c0e2123e7f..5737df6e31 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -1319,6 +1319,7 @@ bool JsonParser::check_and_log_curl_error( Path const& path, std::shared_ptr reader ) { +#if !CLP_S_STATIC_EXE if (auto network_reader = std::dynamic_pointer_cast(reader); nullptr != network_reader) { @@ -1335,6 +1336,7 @@ bool JsonParser::check_and_log_curl_error( return true; } } +#endif return false; } } // namespace clp_s diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index cdb16f9c9b..f3d33ad781 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -297,7 +297,9 @@ int main(int argc, char const* argv[]) { clp_s::TimestampPattern::init(); mongocxx::instance const mongocxx_instance{}; +#if !CLP_S_STATIC_EXE clp::CurlGlobalInstance const curl_instance{}; +#endif CommandLineArguments command_line_arguments("clp-s"); auto parsing_result = command_line_arguments.parse_arguments(argc, argv); From bd0ad0aeaf01a5aa74849f2105db037b7f72a14d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 10:57:14 -0500 Subject: [PATCH 02/31] Remove dep on shared mongocxx --- components/core/src/clp_s/CMakeLists.txt | 5 +++++ components/core/src/clp_s/JsonConstructor.cpp | 4 ++++ components/core/src/clp_s/OutputHandlerImpl.cpp | 6 ++++++ components/core/src/clp_s/OutputHandlerImpl.hpp | 2 ++ components/core/src/clp_s/clp-s.cpp | 4 ++-- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 8a9e3f966a..d55018a000 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -377,6 +377,11 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${CLP_S_JSON_CONSTRUCTOR_SOURCES} ) add_library(clp_s::json_constructor ALIAS clp_s_json_constructor) + target_compile_definitions( + clp_s_json_constructor + PRIVATE + CLP_S_STATIC_EXE=$ + ) target_compile_features(clp_s_json_constructor PRIVATE cxx_std_20) target_include_directories(clp_s_json_constructor PUBLIC ../) target_link_libraries( diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 67f66c0c4c..3d37701634 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -62,6 +62,9 @@ void JsonConstructor::store() { } void JsonConstructor::construct_in_order() { +#if CLP_S_STATIC_EXE + throw OperationFailed(ErrorCodeUnsupported, __FILE__, __LINE__, "in order construct"); +#else std::string buffer; auto tables = m_archive_reader->read_all_tables(); using ReaderPointer = std::shared_ptr; @@ -188,5 +191,6 @@ void JsonConstructor::construct_in_order() { throw OperationFailed(ErrorCodeFailureDbBulkWrite, __FILE__, __LINE__, e.what()); } } +#endif } } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index ba6d6fbb41..0525a7acdc 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -75,6 +75,9 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( : ::clp_s::search::OutputHandler(should_output_timestamp, true), m_batch_size(batch_size), m_max_num_results(max_num_results) { +#if CLP_S_STATIC_EXE + throw OperationFailed(ErrorCode::ErrorCodeUnsupported, __FILENAME__, __LINE__); +#else try { auto mongo_uri = mongocxx::uri(uri); m_client = mongocxx::client(mongo_uri); @@ -83,9 +86,11 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( } catch (mongocxx::exception const& e) { throw OperationFailed(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); } +#endif } ErrorCode ResultsCacheOutputHandler::flush() { +#if !CLP_S_STATIC_EXE size_t count = 0; while (false == m_latest_results.empty()) { auto result = std::move(*m_latest_results.top()); @@ -138,6 +143,7 @@ ErrorCode ResultsCacheOutputHandler::flush() { } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } +#endif return ErrorCode::ErrorCodeSuccess; } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 510df924cb..19ba2bb339 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -186,9 +186,11 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { void write(std::string_view message) override { write(message, 0, {}, 0); } private: +#if !CLP_S_STATIC_EXE mongocxx::client m_client; mongocxx::collection m_collection; std::vector m_results; +#endif uint64_t m_batch_size; uint64_t m_max_num_results; std::priority_queue< diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index f3d33ad781..6258dedb3f 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -8,7 +8,7 @@ #include #include -#include +//#include #include #include #include @@ -296,8 +296,8 @@ int main(int argc, char const* argv[]) { } clp_s::TimestampPattern::init(); - mongocxx::instance const mongocxx_instance{}; #if !CLP_S_STATIC_EXE + mongocxx::instance const mongocxx_instance{}; clp::CurlGlobalInstance const curl_instance{}; #endif From 25763a83fd0fee4ffc807814113f88c9a10c2bec Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 10:58:51 -0500 Subject: [PATCH 03/31] Remove dep on libzstd.so --- components/core/src/clp_s/CMakeLists.txt | 4 ++-- components/core/src/clp_s/JsonConstructor.cpp | 8 ++++---- components/core/src/clp_s/OutputHandlerImpl.cpp | 10 +++++----- components/core/src/clp_s/OutputHandlerImpl.hpp | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index d55018a000..9576be3327 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -391,7 +391,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${zstd_TARGET} PRIVATE fmt::fmt - ${MONGOCXX_TARGET} + #${MONGOCXX_TARGET} spdlog::spdlog ) endif() @@ -459,7 +459,7 @@ if(CLP_BUILD_EXECUTABLES) clp_s::search clp_s::search::ast clp_s::search::kql - ${MONGOCXX_TARGET} + #${MONGOCXX_TARGET} msgpack-cxx spdlog::spdlog ystdlib::error_handling diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 3d37701634..568687b8f3 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -5,10 +5,10 @@ #include #include -#include -#include -#include -#include +//#include +//#include +//#include +//#include #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 0525a7acdc..de3163e828 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -4,11 +4,11 @@ #include #include -#include -#include -#include -#include -#include +//#include +//#include +//#include +//#include +//#include #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 19ba2bb339..4be79bf382 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -11,8 +11,8 @@ #include #include -#include -#include +//#include +//#include #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" From 70307ed09c4d2dedecdec3a24047f6a4f0d59ab3 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 11:34:02 -0500 Subject: [PATCH 04/31] Restore in order decompression functionality --- components/core/src/clp_s/JsonConstructor.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 568687b8f3..82b6d874ec 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -62,9 +62,6 @@ void JsonConstructor::store() { } void JsonConstructor::construct_in_order() { -#if CLP_S_STATIC_EXE - throw OperationFailed(ErrorCodeUnsupported, __FILE__, __LINE__, "in order construct"); -#else std::string buffer; auto tables = m_archive_reader->read_all_tables(); using ReaderPointer = std::shared_ptr; @@ -83,6 +80,7 @@ void JsonConstructor::construct_in_order() { FileWriter writer; writer.open(src_path, FileWriter::OpenMode::CreateForWriting); +#if !CLP_S_STATIC_EXE mongocxx::client client; mongocxx::collection collection; @@ -97,6 +95,7 @@ void JsonConstructor::construct_in_order() { } std::vector results; +#endif auto finalize_chunk = [&](bool open_new_writer) { // Add one to last_idx to match clp's behaviour of having the end index be exclusive ++last_idx; @@ -110,6 +109,7 @@ void JsonConstructor::construct_in_order() { throw OperationFailed(ErrorCodeFailure, __FILE__, __LINE__, ec.message()); } +#if !CLP_S_STATIC_EXE if (m_option.metadata_db.has_value()) { results.emplace_back( std::move( @@ -138,6 +138,7 @@ void JsonConstructor::construct_in_order() { ) ); } +#endif if (m_option.print_ordered_chunk_stats) { nlohmann::json json_msg; @@ -184,6 +185,7 @@ void JsonConstructor::construct_in_order() { } } +#if !CLP_S_STATIC_EXE if (false == results.empty()) { try { collection.insert_many(results); From 6604839e31d25aab03730b8eab0ca6dfc519de31 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 13:09:55 -0500 Subject: [PATCH 05/31] Revert "Remove dep on libzstd.so" This reverts commit 25763a83fd0fee4ffc807814113f88c9a10c2bec. --- components/core/src/clp_s/CMakeLists.txt | 4 ++-- components/core/src/clp_s/JsonConstructor.cpp | 8 ++++---- components/core/src/clp_s/OutputHandlerImpl.cpp | 10 +++++----- components/core/src/clp_s/OutputHandlerImpl.hpp | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 9576be3327..d55018a000 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -391,7 +391,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${zstd_TARGET} PRIVATE fmt::fmt - #${MONGOCXX_TARGET} + ${MONGOCXX_TARGET} spdlog::spdlog ) endif() @@ -459,7 +459,7 @@ if(CLP_BUILD_EXECUTABLES) clp_s::search clp_s::search::ast clp_s::search::kql - #${MONGOCXX_TARGET} + ${MONGOCXX_TARGET} msgpack-cxx spdlog::spdlog ystdlib::error_handling diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 82b6d874ec..f72322b180 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -5,10 +5,10 @@ #include #include -//#include -//#include -//#include -//#include +#include +#include +#include +#include #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index de3163e828..0525a7acdc 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -4,11 +4,11 @@ #include #include -//#include -//#include -//#include -//#include -//#include +#include +#include +#include +#include +#include #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 4be79bf382..19ba2bb339 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -11,8 +11,8 @@ #include #include -//#include -//#include +#include +#include #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" From 920db8cdbf0329fed24eba76f758ccdf307ba06a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 18:21:26 +0000 Subject: [PATCH 06/31] uncomment --- components/core/src/clp_s/clp-s.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 6258dedb3f..660be3b157 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -8,7 +8,7 @@ #include #include -//#include +#include #include #include #include From bf435ebcaa76cf92c91090e19965fdc04dedc146 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 18:21:47 +0000 Subject: [PATCH 07/31] Point mongocxx to our static libzstd.a --- taskfiles/deps/main.yaml | 8 +++++++- taskfiles/deps/utils.yaml | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 927cedb2a1..3509efb94d 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -22,6 +22,10 @@ vars: G_BOOST_LIB_NAME: "Boost" G_FMT_LIB_NAME: "fmt" G_GSL_LIB_NAME: "Microsoft.GSL" + G_ZSTD_LIB_NAME: "zstd" + + # Installation directories + G_ZSTD_INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.G_ZSTD_LIB_NAME}}-install" # Antlr G_ANTLR_VERSION: "4.13.2" @@ -438,6 +442,7 @@ tasks: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DENABLE_UNINSTALL=OFF" + - "-Dpkgcfg_lib_ZSTD_zstd={{.G_ZSTD_INSTALL_PREFIX}}/lib/libzstd.a" LIB_NAME: "mongocxx" TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ @@ -632,6 +637,7 @@ tasks: - "-DZSTD_BUILD_STATIC=ON" - "-DZSTD_BUILD_TESTS=OFF" CMAKE_SOURCE_DIR: "build/cmake" - LIB_NAME: "zstd" + INSTALL_PREFIX: "{{.G_ZSTD_INSTALL_PREFIX}}" + LIB_NAME: "{{.G_ZSTD_LIB_NAME}}" TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" TARBALL_URL: "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index b6f71720a1..b34f297f7c 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -48,6 +48,7 @@ tasks: # command. # @param {string} [CMAKE_SOURCE_DIR=.] The path, within the tarball extraction directory, # containing the project's top level CMakeLists.txt. + # @param {string} [INSTALL_PREFIX] Path to the installation directory. install-remote-cmake-lib: internal: true vars: @@ -55,7 +56,8 @@ tasks: ref: "default (list) .CMAKE_GEN_ARGS" CMAKE_SOURCE_DIR: >- {{default "." .CMAKE_SOURCE_DIR}} - INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" + INSTALL_PREFIX: >- + {{default "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" .INSTALL_PREFIX}} INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" deps: - task: "init" From 11349b51238e86fd0291dd5bf71fb121cee5c135 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 2 Dec 2025 18:43:09 +0000 Subject: [PATCH 08/31] make mongocxx task depend on zstd --- taskfiles/deps/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 3509efb94d..cb49de9ad2 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -433,6 +433,8 @@ tasks: vars: VERSION: "r4.1.1" run: "once" + deps: + - task: "zstd" cmds: - task: "utils:install-remote-cmake-lib" vars: From 3f0a3672b5bf70dad9bd4db036426c9414b9349a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 4 Dec 2025 08:19:40 +0000 Subject: [PATCH 09/31] Address review comments --- components/core/src/clp_s/CMakeLists.txt | 46 +++++++++++++++++-- components/core/src/clp_s/InputConfig.cpp | 11 +++-- components/core/src/clp_s/JsonConstructor.cpp | 6 +-- components/core/src/clp_s/JsonParser.cpp | 16 ++++++- .../core/src/clp_s/OutputHandlerImpl.cpp | 8 +--- .../core/src/clp_s/OutputHandlerImpl.hpp | 4 +- components/core/src/clp_s/clp-s.cpp | 9 +++- 7 files changed, 77 insertions(+), 23 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index d55018a000..d038c2d3a4 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -1,4 +1,34 @@ -option(CLP_S_STATIC_EXE "Enable clp-s static exe" OFF) +include(CMakeDependentOption) + +option( + CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE + "Build clp-s as a simplified statically linked executable with reduced dependencies." + OFF +) + +cmake_dependent_option( + CLP_S_EXCLUDE_LIBCURL + "Exclude libcurl support from the simplified static clp-s executable." + ON + CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE + OFF +) + +cmake_dependent_option( + CLP_S_EXCLUDE_MONGOCXX + "Exclude mongocxx support from the simplified static clp-s executable." + ON + CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE + OFF +) + +cmake_dependent_option( + CLP_S_STATIC_GCC_RUNTIME + "Link the simplified static clp-s executable against static libgcc and libstdc++." + ON + CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE + OFF +) add_subdirectory(indexer) add_subdirectory(log_converter) @@ -215,7 +245,7 @@ if(CLP_BUILD_CLP_S_IO) target_compile_definitions( clp_s_io PRIVATE - CLP_S_STATIC_EXE=$ + CLP_S_EXCLUDE_LIBCURL=$ ) target_compile_features(clp_s_io PRIVATE cxx_std_20) target_include_directories(clp_s_io PUBLIC ../) @@ -278,6 +308,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER) ${CLP_S_ARCHIVE_WRITER_SOURCES} ) add_library(clp_s::archive_writer ALIAS clp_s_archive_writer) + target_compile_definitions( + clp_s_archive_writer + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp_s_archive_writer PRIVATE cxx_std_20) target_include_directories(clp_s_archive_writer PUBLIC ../) target_link_libraries( @@ -380,7 +415,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) target_compile_definitions( clp_s_json_constructor PRIVATE - CLP_S_STATIC_EXE=$ + CLP_S_EXCLUDE_MONGOCXX=$ ) target_compile_features(clp_s_json_constructor PRIVATE cxx_std_20) target_include_directories(clp_s_json_constructor PUBLIC ../) @@ -443,7 +478,8 @@ if(CLP_BUILD_EXECUTABLES) target_compile_definitions( clp-s PRIVATE - CLP_S_STATIC_EXE=$ + CLP_S_EXCLUDE_LIBCURL=$ + CLP_S_EXCLUDE_MONGOCXX=$ ) target_compile_features(clp-s PRIVATE cxx_std_20) target_link_libraries( @@ -469,7 +505,7 @@ if(CLP_BUILD_EXECUTABLES) PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}" ) - if(CLP_S_STATIC_EXE) + if(CLP_S_STATIC_GCC_RUNTIME) target_link_options( clp-s PRIVATE diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index d0d10cc1cb..24557c1145 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -191,12 +192,14 @@ auto try_sign_url(std::string& url) -> bool { return true; } +#if CLP_S_EXCLUDE_LIBCURL auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) -> std::shared_ptr { -#if CLP_S_STATIC_EXE - SPDLOG_ERROR("Static clp-s executable currently does not support NetworkReader."); - return nullptr; + throw std::runtime_error("Simplified static clp-s executable does not support libcurl."); +} #else +auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) + -> std::shared_ptr { std::string request_url{url}; switch (auth.method) { case AuthMethod::S3PresignedUrlV4: @@ -216,8 +219,8 @@ auto try_create_network_reader(std::string_view const url, NetworkAuthOption con SPDLOG_ERROR("Failed to open url for reading - {}", e.what()); return nullptr; } -#endif } +#endif auto could_be_zstd(char const* peek_buf, size_t peek_size) -> bool { constexpr std::array cZstdMagicNumber = {'\x28', '\xB5', '\x2F', '\xFD'}; diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index f72322b180..304e5cf0ea 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -80,7 +80,7 @@ void JsonConstructor::construct_in_order() { FileWriter writer; writer.open(src_path, FileWriter::OpenMode::CreateForWriting); -#if !CLP_S_STATIC_EXE +#if !CLP_S_EXCLUDE_MONGOCXX mongocxx::client client; mongocxx::collection collection; @@ -109,7 +109,7 @@ void JsonConstructor::construct_in_order() { throw OperationFailed(ErrorCodeFailure, __FILE__, __LINE__, ec.message()); } -#if !CLP_S_STATIC_EXE +#if !CLP_S_EXCLUDE_MONGOCXX if (m_option.metadata_db.has_value()) { results.emplace_back( std::move( @@ -185,7 +185,7 @@ void JsonConstructor::construct_in_order() { } } -#if !CLP_S_STATIC_EXE +#if !CLP_S_EXCLUDE_MONGOCXX if (false == results.empty()) { try { collection.insert_many(results); diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 5737df6e31..89b06de4ed 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1315,11 +1316,22 @@ void JsonParser::split_archive() { m_archive_writer->open(m_archive_options); } +#if CLP_S_EXCLUDE_LIBCURL +bool JsonParser::check_and_log_curl_error( + Path const& path, + std::shared_ptr reader +) { + if (auto network_reader = std::dynamic_pointer_cast(reader); + nullptr != network_reader) + { + throw std::runtime_error("Simplified static clp-s executable does not support libcurl."); + } +} +#else bool JsonParser::check_and_log_curl_error( Path const& path, std::shared_ptr reader ) { -#if !CLP_S_STATIC_EXE if (auto network_reader = std::dynamic_pointer_cast(reader); nullptr != network_reader) { @@ -1336,7 +1348,7 @@ bool JsonParser::check_and_log_curl_error( return true; } } -#endif return false; } +#endif } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 0525a7acdc..85866edbfc 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -65,6 +65,7 @@ void NetworkOutputHandler::write( } } +#if !CLP_S_EXCLUDE_MONGOCXX ResultsCacheOutputHandler::ResultsCacheOutputHandler( string const& uri, string const& collection, @@ -75,9 +76,6 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( : ::clp_s::search::OutputHandler(should_output_timestamp, true), m_batch_size(batch_size), m_max_num_results(max_num_results) { -#if CLP_S_STATIC_EXE - throw OperationFailed(ErrorCode::ErrorCodeUnsupported, __FILENAME__, __LINE__); -#else try { auto mongo_uri = mongocxx::uri(uri); m_client = mongocxx::client(mongo_uri); @@ -86,11 +84,9 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( } catch (mongocxx::exception const& e) { throw OperationFailed(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); } -#endif } ErrorCode ResultsCacheOutputHandler::flush() { -#if !CLP_S_STATIC_EXE size_t count = 0; while (false == m_latest_results.empty()) { auto result = std::move(*m_latest_results.top()); @@ -143,7 +139,6 @@ ErrorCode ResultsCacheOutputHandler::flush() { } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } -#endif return ErrorCode::ErrorCodeSuccess; } @@ -176,6 +171,7 @@ void ResultsCacheOutputHandler::write( ); } } +#endif CountOutputHandler::CountOutputHandler(int reducer_socket_fd) : ::clp_s::search::OutputHandler(false, false), diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 19ba2bb339..23bdb2a37d 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -115,6 +115,7 @@ class NetworkOutputHandler : public ::clp_s::search::OutputHandler { int m_socket_fd; }; +#if !CLP_S_EXCLUDE_MONGOCXX /** * Output handler that writes to a MongoDB collection. */ @@ -186,11 +187,9 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { void write(std::string_view message) override { write(message, 0, {}, 0); } private: -#if !CLP_S_STATIC_EXE mongocxx::client m_client; mongocxx::collection m_collection; std::vector m_results; -#endif uint64_t m_batch_size; uint64_t m_max_num_results; std::priority_queue< @@ -200,6 +199,7 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { > m_latest_results; }; +#endif /** * Output handler that performs a count aggregation and sends the results to a reducer. diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 660be3b157..49ca30f888 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -254,6 +255,9 @@ bool search_archive( } break; case CommandLineArguments::OutputHandlerType::ResultsCache: +#if CLP_S_EXCLUDE_MONGOCXX + throw std::runtime_error("Simplified static clp-s executable does not support mongocxx."); +#else output_handler = std::make_unique( command_line_arguments.get_mongodb_uri(), command_line_arguments.get_mongodb_collection(), @@ -261,6 +265,7 @@ bool search_archive( command_line_arguments.get_max_num_results() ); break; +#endif case CommandLineArguments::OutputHandlerType::Stdout: output_handler = std::make_unique(); break; @@ -296,8 +301,10 @@ int main(int argc, char const* argv[]) { } clp_s::TimestampPattern::init(); -#if !CLP_S_STATIC_EXE +#if !CLP_S_EXCLUDE_MONGOCXX mongocxx::instance const mongocxx_instance{}; +#endif +#if !CLP_S_EXCLUDE_LIBCURL clp::CurlGlobalInstance const curl_instance{}; #endif From c06a3397966a8564fe956c28b4382d6af1dcc3a4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 4 Dec 2025 09:06:40 +0000 Subject: [PATCH 10/31] Add missing return statement --- components/core/src/clp_s/JsonParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 89b06de4ed..4bdd3fa7b6 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -1326,6 +1326,7 @@ bool JsonParser::check_and_log_curl_error( { throw std::runtime_error("Simplified static clp-s executable does not support libcurl."); } + return false; } #else bool JsonParser::check_and_log_curl_error( From 21a96be4200730518b7c2c73cd34b42cf6df3957 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 4 Dec 2025 09:25:24 +0000 Subject: [PATCH 11/31] lint fix --- components/core/src/clp_s/clp-s.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 49ca30f888..daded90588 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -3,8 +3,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -256,7 +256,9 @@ bool search_archive( break; case CommandLineArguments::OutputHandlerType::ResultsCache: #if CLP_S_EXCLUDE_MONGOCXX - throw std::runtime_error("Simplified static clp-s executable does not support mongocxx."); + throw std::runtime_error( + "Simplified static clp-s executable does not support mongocxx." + ); #else output_handler = std::make_unique( command_line_arguments.get_mongodb_uri(), From 6520e0c3440d13e66123bf090c24ba949aa3d0f4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 4 Dec 2025 10:53:39 +0000 Subject: [PATCH 12/31] Redirect zstd install lib from lib64 to lib --- taskfiles/deps/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index cb49de9ad2..7d12e335df 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -632,6 +632,7 @@ tasks: vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_LIBDIR=lib" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DZSTD_BUILD_CONTRIB=OFF" - "-DZSTD_BUILD_PROGRAMS=OFF" From 6b73db09820166b6f7070a20d7729dec16636d62 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 00:02:15 +0000 Subject: [PATCH 13/31] Link against cmake targets conditionally --- components/core/src/clp_s/CMakeLists.txt | 48 ++++++++++++++++--- components/core/src/clp_s/JsonConstructor.cpp | 2 + components/core/src/clp_s/JsonParser.cpp | 2 + .../core/src/clp_s/OutputHandlerImpl.cpp | 2 + .../core/src/clp_s/OutputHandlerImpl.hpp | 2 + components/core/src/clp_s/clp-s.cpp | 4 ++ 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index d038c2d3a4..3b6671ae73 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -165,14 +165,26 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) ${zstd_TARGET} PRIVATE Boost::regex - ${CURL_LIBRARIES} fmt::fmt msgpack-cxx nlohmann_json::nlohmann_json - OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling ) + if(NOT CLP_S_EXCLUDE_LIBCURL) + target_link_libraries( + clp_s_clp_dependencies + PRIVATE + ${CURL_LIBRARIES} + ) + endif() + if(NOT CLP_S_EXCLUDE_MONGOCXX) + target_link_libraries( + clp_s_clp_dependencies + PRIVATE + OpenSSL::Crypto + ) + endif() endif() set( @@ -328,10 +340,16 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER) PRIVATE Boost::url clp_s::timestamp_pattern - ${CURL_LIBRARIES} fmt::fmt spdlog::spdlog ) + if(NOT CLP_S_EXCLUDE_LIBCURL) + target_link_libraries( + clp_s_archive_writer + PRIVATE + ${CURL_LIBRARIES} + ) + endif() endif() set( @@ -392,10 +410,16 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) Boost::url clp_s::clp_dependencies clp_s::timestamp_pattern - ${CURL_LIBRARIES} fmt::fmt spdlog::spdlog ) + if(NOT CLP_S_EXCLUDE_LIBCURL) + target_link_libraries( + clp_s_archive_writer + PRIVATE + ${CURL_LIBRARIES} + ) + endif() endif() set( @@ -426,9 +450,15 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${zstd_TARGET} PRIVATE fmt::fmt - ${MONGOCXX_TARGET} spdlog::spdlog ) + if(NOT CLP_S_EXCLUDE_MONGOCXX) + target_link_libraries( + clp_s_clp_dependencies + PRIVATE + ${MONGOCXX_TARGET} + ) + endif() endif() set( @@ -495,11 +525,17 @@ if(CLP_BUILD_EXECUTABLES) clp_s::search clp_s::search::ast clp_s::search::kql - ${MONGOCXX_TARGET} msgpack-cxx spdlog::spdlog ystdlib::error_handling ) + if(NOT CLP_S_EXCLUDE_MONGOCXX) + target_link_libraries( + clp-s + PRIVATE + ${MONGOCXX_TARGET} + ) + endif() set_target_properties( clp-s PROPERTIES diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 304e5cf0ea..11b97f0404 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -5,10 +5,12 @@ #include #include +#if !CLP_S_EXCLUDE_MONGOCXX #include #include #include #include +#endif #include #include diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 4bdd3fa7b6..8c7911dcc3 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -14,7 +14,9 @@ #include #include +#if !CLP_S_EXCLUDE_LIBCURL #include +#endif #include #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 85866edbfc..062317082b 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -4,11 +4,13 @@ #include #include +#if !CLP_S_EXCLUDE_MONGOCXX #include #include #include #include #include +#endif #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 23bdb2a37d..0a49c2a390 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -11,8 +11,10 @@ #include #include +#if !CLP_S_EXCLUDE_MONGOCXX #include #include +#endif #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index daded90588..0bcc682abe 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -9,12 +9,16 @@ #include #include +#if !CLP_S_EXCLUDE_MONGOCXX #include +#endif #include #include #include +#if !CLP_S_EXCLUDE_LIBCURL #include "../clp/CurlGlobalInstance.hpp" +#endif #include "../clp/ir/constants.hpp" #include "../clp/streaming_archive/ArchiveMetadata.hpp" #include "../reducer/network_utils.hpp" From ba9a2c95ecd6a9061586909fa167c6f7cf5e3fb1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 00:28:29 +0000 Subject: [PATCH 14/31] Remove unused zstd targets --- components/core/src/clp_s/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 3b6671ae73..9923eeb455 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -162,7 +162,6 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) clp::string_utils log_surgeon::log_surgeon ystdlib::containers - ${zstd_TARGET} PRIVATE Boost::regex fmt::fmt @@ -447,7 +446,6 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) clp_s_json_constructor PUBLIC clp_s::archive_reader - ${zstd_TARGET} PRIVATE fmt::fmt spdlog::spdlog From c8f3e8697d21112fbf2ea65ee4057501476c2a1a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 03:47:32 +0000 Subject: [PATCH 15/31] Bug fix --- components/core/src/clp_s/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 9923eeb455..4f5a78c81d 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -414,7 +414,7 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) ) if(NOT CLP_S_EXCLUDE_LIBCURL) target_link_libraries( - clp_s_archive_writer + clp_s_archive_reader PRIVATE ${CURL_LIBRARIES} ) @@ -452,7 +452,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ) if(NOT CLP_S_EXCLUDE_MONGOCXX) target_link_libraries( - clp_s_clp_dependencies + clp_s_json_constructor PRIVATE ${MONGOCXX_TARGET} ) From abd8616b159d3df98a8973fb143b7964e25ceed4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 04:12:02 +0000 Subject: [PATCH 16/31] lint frix --- components/core/src/clp_s/JsonConstructor.cpp | 8 ++++---- components/core/src/clp_s/JsonParser.cpp | 2 +- components/core/src/clp_s/OutputHandlerImpl.cpp | 10 +++++----- components/core/src/clp_s/OutputHandlerImpl.hpp | 4 ++-- components/core/src/clp_s/clp-s.cpp | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 11b97f0404..bff8503983 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -6,10 +6,10 @@ #include #if !CLP_S_EXCLUDE_MONGOCXX -#include -#include -#include -#include + #include + #include + #include + #include #endif #include #include diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 8c7911dcc3..d859c67878 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -15,7 +15,7 @@ #include #include #if !CLP_S_EXCLUDE_LIBCURL -#include + #include #endif #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 062317082b..0ac475d54a 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -5,11 +5,11 @@ #include #if !CLP_S_EXCLUDE_MONGOCXX -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include #endif #include #include diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 0a49c2a390..1277185a3f 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -12,8 +12,8 @@ #include #if !CLP_S_EXCLUDE_MONGOCXX -#include -#include + #include + #include #endif #include "../reducer/Pipeline.hpp" diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 0bcc682abe..7155690eff 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -10,14 +10,14 @@ #include #if !CLP_S_EXCLUDE_MONGOCXX -#include + #include #endif #include #include #include #if !CLP_S_EXCLUDE_LIBCURL -#include "../clp/CurlGlobalInstance.hpp" + #include "../clp/CurlGlobalInstance.hpp" #endif #include "../clp/ir/constants.hpp" #include "../clp/streaming_archive/ArchiveMetadata.hpp" From 0177422edb0c87b13ae2eb80856b44bcb70d9203 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 04:19:42 +0000 Subject: [PATCH 17/31] Add task to build static clp-s --- taskfile.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 118fd6fb6c..ce179b383d 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -24,6 +24,7 @@ vars: G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CONFIG_SCHEMA_BUILD_DIR: "{{.G_BUILD_DIR}}/config-schema" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" + G_CORE_STATIC_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core-static" G_NODEJS_22_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-22" G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" @@ -206,6 +207,44 @@ tasks: JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" TARGETS: ["clg", "clo", "clp", "clp-s", "indexer", "log-converter", "reducer-server"] + core-static: + cmds: + - task: "core-static-generate" + - task: "core-static-build" + + core-static-generate: + internal: true + sources: *core_source_files + deps: + - "codegen:clp-s-generate-parsers" + - "deps:core" + cmds: + - task: "utils:cmake:generate" + vars: + BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" + EXTRA_ARGS: + - "-DCLP_USE_STATIC_LIBS=ON" + - "-DCLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE=ON" + SOURCE_DIR: "{{.G_CORE_COMPONENT_DIR}}" + + core-static-build: + internal: true + sources: *core_source_files + generates: + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clg" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clo" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/log-converter" + - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" + cmds: + - task: "utils:cmake:build" + vars: + BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" + JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" + TARGETS: ["clg", "clo", "clp", "clp-s", "indexer", "log-converter", "reducer-server"] + # NOTE: We rely on cargo for checksumming. It automatically rebuilds when source files change or # when a binary is missing. rust: From a68ffb95ae58ebf3639082ded00c1b6546c0cead Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 04:37:16 +0000 Subject: [PATCH 18/31] reduce workflow --- taskfile.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index ce179b383d..61010a1f93 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -216,34 +216,27 @@ tasks: internal: true sources: *core_source_files deps: - - "codegen:clp-s-generate-parsers" - "deps:core" cmds: - task: "utils:cmake:generate" vars: BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" EXTRA_ARGS: - - "-DCLP_USE_STATIC_LIBS=ON" - "-DCLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE=ON" + - "-DCLP_USE_STATIC_LIBS=ON" SOURCE_DIR: "{{.G_CORE_COMPONENT_DIR}}" core-static-build: internal: true sources: *core_source_files generates: - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clg" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clo" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/log-converter" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" + - "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}/clp-s" cmds: - task: "utils:cmake:build" vars: BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - TARGETS: ["clg", "clo", "clp", "clp-s", "indexer", "log-converter", "reducer-server"] + TARGETS: ["clp-s"] # NOTE: We rely on cargo for checksumming. It automatically rebuilds when source files change or # when a binary is missing. From 7e1793d3c7f3c6ac85e4bc86e273b4b7c91e030c Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 05:05:25 +0000 Subject: [PATCH 19/31] Add back missing zstd target --- components/core/src/clp_s/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 4f5a78c81d..780ce0d2c8 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -162,6 +162,7 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) clp::string_utils log_surgeon::log_surgeon ystdlib::containers + ${zstd_TARGET} PRIVATE Boost::regex fmt::fmt From c8b0444bcd45324136352277176e49556699734f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 5 Dec 2025 05:53:44 +0000 Subject: [PATCH 20/31] Address coderabbit review --- components/core/src/clp_s/CMakeLists.txt | 6 ++++++ components/core/src/clp_s/InputConfig.cpp | 4 +++- components/core/src/clp_s/JsonParser.cpp | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 780ce0d2c8..0b871abaec 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -395,6 +395,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) ${CLP_S_ARCHIVE_READER_SOURCES} ) add_library(clp_s::archive_reader ALIAS clp_s_archive_reader) + target_compile_definitions( + clp_s_archive_reader + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp_s_archive_reader PRIVATE cxx_std_20) target_include_directories(clp_s_archive_reader PUBLIC ../) target_link_libraries( @@ -447,6 +452,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) clp_s_json_constructor PUBLIC clp_s::archive_reader + ${zstd_TARGET} PRIVATE fmt::fmt spdlog::spdlog diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index 24557c1145..2ca1c06408 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -8,7 +8,9 @@ #include #include #include -#include +#if CLP_S_EXCLUDE_LIBCURL + #include +#endif #include #include diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index d859c67878..f28dfde5de 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -6,7 +6,9 @@ #include #include #include -#include +#if CLP_S_EXCLUDE_LIBCURL + #include +#endif #include #include #include From 150a97496ca51feaf24eac595dfca335708f06f8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 26 Feb 2026 16:03:44 -0500 Subject: [PATCH 21/31] Fix merge error --- taskfiles/deps/main.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 64c8f3324a..f302aaf4cb 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -22,14 +22,11 @@ vars: G_BOOST_LIB_NAME: "Boost" G_FMT_LIB_NAME: "fmt" G_GSL_LIB_NAME: "Microsoft.GSL" -<<<<<<< HEAD G_ZSTD_LIB_NAME: "zstd" # Installation directories G_ZSTD_INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.G_ZSTD_LIB_NAME}}-install" -======= G_YSTDLIB_LIB_NAME: "ystdlib" ->>>>>>> main # Antlr G_ANTLR_VERSION: "4.13.2" From e903aa7452a0ab070f7ddb4423053d8155dc364d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 26 Feb 2026 16:08:22 -0500 Subject: [PATCH 22/31] Remove taskfile changes --- taskfile.yaml | 32 -------------------------------- taskfiles/deps/main.yaml | 11 +---------- taskfiles/deps/utils.yaml | 4 +--- 3 files changed, 2 insertions(+), 45 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index eeefdf6a40..9bd957497c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -26,7 +26,6 @@ vars: G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CONFIG_SCHEMA_BUILD_DIR: "{{.G_BUILD_DIR}}/config-schema" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" - G_CORE_STATIC_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core-static" G_NODEJS_22_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-22" G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" @@ -207,37 +206,6 @@ tasks: JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" TARGETS: ["clg", "clo", "clp", "clp-s", "indexer", "log-converter", "reducer-server"] - core-static: - cmds: - - task: "core-static-generate" - - task: "core-static-build" - - core-static-generate: - internal: true - sources: *core_source_files - deps: - - "deps:core" - cmds: - - task: "utils:cmake:generate" - vars: - BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" - EXTRA_ARGS: - - "-DCLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE=ON" - - "-DCLP_USE_STATIC_LIBS=ON" - SOURCE_DIR: "{{.G_CORE_COMPONENT_DIR}}" - - core-static-build: - internal: true - sources: *core_source_files - generates: - - "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}/clp-s" - cmds: - - task: "utils:cmake:build" - vars: - BUILD_DIR: "{{.G_CORE_STATIC_COMPONENT_BUILD_DIR}}" - JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - TARGETS: ["clp-s"] - # NOTE: We rely on cargo for checksumming. It automatically rebuilds when source files change or # when a binary is missing. rust: diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index f302aaf4cb..788c03c298 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -22,10 +22,6 @@ vars: G_BOOST_LIB_NAME: "Boost" G_FMT_LIB_NAME: "fmt" G_GSL_LIB_NAME: "Microsoft.GSL" - G_ZSTD_LIB_NAME: "zstd" - - # Installation directories - G_ZSTD_INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.G_ZSTD_LIB_NAME}}-install" G_YSTDLIB_LIB_NAME: "ystdlib" # Antlr @@ -422,8 +418,6 @@ tasks: vars: VERSION: "r4.1.1" run: "once" - deps: - - task: "zstd" cmds: - task: "utils:install-remote-cmake-lib" vars: @@ -433,7 +427,6 @@ tasks: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DENABLE_UNINSTALL=OFF" - - "-Dpkgcfg_lib_ZSTD_zstd={{.G_ZSTD_INSTALL_PREFIX}}/lib/libzstd.a" LIB_NAME: "mongocxx" TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ @@ -621,7 +614,6 @@ tasks: vars: CMAKE_GEN_ARGS: - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_LIBDIR=lib" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DZSTD_BUILD_CONTRIB=OFF" - "-DZSTD_BUILD_PROGRAMS=OFF" @@ -629,7 +621,6 @@ tasks: - "-DZSTD_BUILD_STATIC=ON" - "-DZSTD_BUILD_TESTS=OFF" CMAKE_SOURCE_DIR: "build/cmake" - INSTALL_PREFIX: "{{.G_ZSTD_INSTALL_PREFIX}}" - LIB_NAME: "{{.G_ZSTD_LIB_NAME}}" + LIB_NAME: "zstd" TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" TARBALL_URL: "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index e842c3c3ff..737e9f6f53 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -48,7 +48,6 @@ tasks: # command. # @param {string} [CMAKE_SOURCE_DIR=.] The path, within the tarball extraction directory, # containing the project's top level CMakeLists.txt. - # @param {string} [INSTALL_PREFIX] Path to the installation directory. install-remote-cmake-lib: internal: true vars: @@ -56,8 +55,7 @@ tasks: ref: "default (list) .CMAKE_GEN_ARGS" CMAKE_SOURCE_DIR: >- {{default "." .CMAKE_SOURCE_DIR}} - INSTALL_PREFIX: >- - {{default "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" .INSTALL_PREFIX}} + INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/{{.LIB_NAME}}-install" INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CPP_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" deps: - task: "init" From 691a3e8e180467fb06c5c1b3ea03c56c0e46ffd7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 26 Feb 2026 17:38:51 -0500 Subject: [PATCH 23/31] revert CLP_S_EXCLUDE_MONGOCXX changes --- components/core/src/clp_s/CMakeLists.txt | 50 ++++++------------- components/core/src/clp_s/JsonConstructor.cpp | 16 ++---- .../core/src/clp_s/OutputHandlerImpl.cpp | 14 ++---- .../core/src/clp_s/OutputHandlerImpl.hpp | 8 +-- components/core/src/clp_s/clp-s.cpp | 12 +---- 5 files changed, 27 insertions(+), 73 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index c8ec7576ed..0674956540 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -14,14 +14,6 @@ cmake_dependent_option( OFF ) -cmake_dependent_option( - CLP_S_EXCLUDE_MONGOCXX - "Exclude mongocxx support from the simplified static clp-s executable." - ON - CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE - OFF -) - cmake_dependent_option( CLP_S_STATIC_GCC_RUNTIME "Link the simplified static clp-s executable against static libgcc and libstdc++." @@ -185,13 +177,11 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) ${CURL_LIBRARIES} ) endif() - if(NOT CLP_S_EXCLUDE_MONGOCXX) - target_link_libraries( - clp_s_clp_dependencies - PRIVATE - OpenSSL::Crypto - ) - endif() + target_link_libraries( + clp_s_clp_dependencies + PRIVATE + OpenSSL::Crypto + ) endif() set( @@ -451,11 +441,6 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${CLP_S_JSON_CONSTRUCTOR_SOURCES} ) add_library(clp_s::json_constructor ALIAS clp_s_json_constructor) - target_compile_definitions( - clp_s_json_constructor - PRIVATE - CLP_S_EXCLUDE_MONGOCXX=$ - ) target_compile_features(clp_s_json_constructor PRIVATE cxx_std_20) target_include_directories(clp_s_json_constructor PUBLIC ../) target_link_libraries( @@ -467,13 +452,11 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) fmt::fmt spdlog::spdlog ) - if(NOT CLP_S_EXCLUDE_MONGOCXX) - target_link_libraries( - clp_s_json_constructor - PRIVATE - ${MONGOCXX_TARGET} - ) - endif() + target_link_libraries( + clp_s_json_constructor + PRIVATE + ${MONGOCXX_TARGET} + ) endif() set( @@ -524,7 +507,6 @@ if(CLP_BUILD_EXECUTABLES) clp-s PRIVATE CLP_S_EXCLUDE_LIBCURL=$ - CLP_S_EXCLUDE_MONGOCXX=$ ) target_compile_features(clp-s PRIVATE cxx_std_20) target_include_directories(clp-s PRIVATE ../) @@ -545,13 +527,11 @@ if(CLP_BUILD_EXECUTABLES) spdlog::spdlog ystdlib::error_handling ) - if(NOT CLP_S_EXCLUDE_MONGOCXX) - target_link_libraries( - clp-s - PRIVATE - ${MONGOCXX_TARGET} - ) - endif() + target_link_libraries( + clp-s + PRIVATE + ${MONGOCXX_TARGET} + ) set_target_properties( clp-s PROPERTIES diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index bff8503983..67f66c0c4c 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -5,12 +5,10 @@ #include #include -#if !CLP_S_EXCLUDE_MONGOCXX - #include - #include - #include - #include -#endif +#include +#include +#include +#include #include #include @@ -82,7 +80,6 @@ void JsonConstructor::construct_in_order() { FileWriter writer; writer.open(src_path, FileWriter::OpenMode::CreateForWriting); -#if !CLP_S_EXCLUDE_MONGOCXX mongocxx::client client; mongocxx::collection collection; @@ -97,7 +94,6 @@ void JsonConstructor::construct_in_order() { } std::vector results; -#endif auto finalize_chunk = [&](bool open_new_writer) { // Add one to last_idx to match clp's behaviour of having the end index be exclusive ++last_idx; @@ -111,7 +107,6 @@ void JsonConstructor::construct_in_order() { throw OperationFailed(ErrorCodeFailure, __FILE__, __LINE__, ec.message()); } -#if !CLP_S_EXCLUDE_MONGOCXX if (m_option.metadata_db.has_value()) { results.emplace_back( std::move( @@ -140,7 +135,6 @@ void JsonConstructor::construct_in_order() { ) ); } -#endif if (m_option.print_ordered_chunk_stats) { nlohmann::json json_msg; @@ -187,7 +181,6 @@ void JsonConstructor::construct_in_order() { } } -#if !CLP_S_EXCLUDE_MONGOCXX if (false == results.empty()) { try { collection.insert_many(results); @@ -195,6 +188,5 @@ void JsonConstructor::construct_in_order() { throw OperationFailed(ErrorCodeFailureDbBulkWrite, __FILE__, __LINE__, e.what()); } } -#endif } } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 0ac475d54a..ba6d6fbb41 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -4,13 +4,11 @@ #include #include -#if !CLP_S_EXCLUDE_MONGOCXX - #include - #include - #include - #include - #include -#endif +#include +#include +#include +#include +#include #include #include @@ -67,7 +65,6 @@ void NetworkOutputHandler::write( } } -#if !CLP_S_EXCLUDE_MONGOCXX ResultsCacheOutputHandler::ResultsCacheOutputHandler( string const& uri, string const& collection, @@ -173,7 +170,6 @@ void ResultsCacheOutputHandler::write( ); } } -#endif CountOutputHandler::CountOutputHandler(int reducer_socket_fd) : ::clp_s::search::OutputHandler(false, false), diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 1277185a3f..510df924cb 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -11,10 +11,8 @@ #include #include -#if !CLP_S_EXCLUDE_MONGOCXX - #include - #include -#endif +#include +#include #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" @@ -117,7 +115,6 @@ class NetworkOutputHandler : public ::clp_s::search::OutputHandler { int m_socket_fd; }; -#if !CLP_S_EXCLUDE_MONGOCXX /** * Output handler that writes to a MongoDB collection. */ @@ -201,7 +198,6 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { > m_latest_results; }; -#endif /** * Output handler that performs a count aggregation and sends the results to a reducer. diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index b7c51f4c34..a9f7c97d05 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -9,9 +9,7 @@ #include #include -#if !CLP_S_EXCLUDE_MONGOCXX - #include -#endif +#include #include #include #include @@ -268,11 +266,6 @@ bool search_archive( } break; case CommandLineArguments::OutputHandlerType::ResultsCache: -#if CLP_S_EXCLUDE_MONGOCXX - throw std::runtime_error( - "Simplified static clp-s executable does not support mongocxx." - ); -#else output_handler = std::make_unique( command_line_arguments.get_mongodb_uri(), command_line_arguments.get_mongodb_collection(), @@ -280,7 +273,6 @@ bool search_archive( command_line_arguments.get_max_num_results() ); break; -#endif case CommandLineArguments::OutputHandlerType::Stdout: output_handler = std::make_unique(); break; @@ -315,9 +307,7 @@ int main(int argc, char const* argv[]) { return 1; } -#if !CLP_S_EXCLUDE_MONGOCXX mongocxx::instance const mongocxx_instance{}; -#endif #if !CLP_S_EXCLUDE_LIBCURL clp::CurlGlobalInstance const curl_instance{}; #endif From bb580f8884a5ace1ec37d23a7dbb14e70a65aa9b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 26 Feb 2026 17:42:02 -0500 Subject: [PATCH 24/31] revert unused options --- components/core/src/clp_s/CMakeLists.txt | 28 +----------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 0674956540..d24be8efaf 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -1,25 +1,7 @@ -include(CMakeDependentOption) - option( - CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE - "Build clp-s as a simplified statically linked executable with reduced dependencies." - OFF -) - -cmake_dependent_option( CLP_S_EXCLUDE_LIBCURL - "Exclude libcurl support from the simplified static clp-s executable." + "Exclude libcurl support from clp-s." ON - CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE - OFF -) - -cmake_dependent_option( - CLP_S_STATIC_GCC_RUNTIME - "Link the simplified static clp-s executable against static libgcc and libstdc++." - ON - CLP_S_BUILD_SIMPLIFIED_STATIC_EXECUTABLE - OFF ) add_subdirectory(indexer) @@ -537,12 +519,4 @@ if(CLP_BUILD_EXECUTABLES) PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}" ) - if(CLP_S_STATIC_GCC_RUNTIME) - target_link_options( - clp-s - PRIVATE - -static-libgcc - -static-libstdc++ - ) - endif() endif() From 4d684dfa4103cec945bfd91abca7340cd37f39c4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 15:10:45 -0500 Subject: [PATCH 25/31] Move check_and_log_curl_error into common clp_s utils --- components/core/src/clp/NetworkReader.hpp | 24 ++++++--- components/core/src/clp_s/JsonParser.cpp | 51 ++----------------- components/core/src/clp_s/JsonParser.hpp | 10 ---- components/core/src/clp_s/Utils.cpp | 30 +++++++++++ components/core/src/clp_s/Utils.hpp | 15 ++++++ .../src/clp_s/log_converter/log_converter.cpp | 35 ++----------- components/core/tests/test-NetworkReader.cpp | 6 +-- 7 files changed, 73 insertions(+), 98 deletions(-) diff --git a/components/core/src/clp/NetworkReader.hpp b/components/core/src/clp/NetworkReader.hpp index 37a9995c93..1a436ae054 100644 --- a/components/core/src/clp/NetworkReader.hpp +++ b/components/core/src/clp/NetworkReader.hpp @@ -66,6 +66,14 @@ class NetworkReader : public ReaderInterface { Finished }; + /** + * CURL error details set by the underlying CURL handler. + */ + struct CurlErrorInfo { + int64_t code; + std::string_view message; + }; + // Constants static constexpr size_t cDefaultBufferPoolSize{8}; static constexpr size_t cDefaultBufferSize{4096}; @@ -224,16 +232,18 @@ class NetworkReader : public ReaderInterface { } /** - * @return The error message set by the underlying CURL handler. - * @return std::nullopt if the download is still in-progress or no error has occured. + * @return CURL error info if the download has completed with a CURL error. + * @return std::nullopt if the download is still in-progress or no error has occurred. */ - [[nodiscard]] auto get_curl_error_msg() const -> std::optional { - if (auto const ret_code{get_curl_ret_code()}; - false == ret_code.has_value() || CURLE_OK == ret_code.value()) - { + [[nodiscard]] auto get_curl_error_info() const -> std::optional { + auto const ret_code = get_curl_ret_code(); + if (false == ret_code.has_value() || CURLcode::CURLE_OK == ret_code.value()) { return std::nullopt; } - return std::string_view{m_curl_error_msg_buf->data()}; + return CurlErrorInfo{ + .code = static_cast(ret_code.value()), + .message = std::string_view{m_curl_error_msg_buf->data()} + }; } private: diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index c4145ab1df..3adc8deb01 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -6,9 +6,6 @@ #include #include #include -#if CLP_S_EXCLUDE_LIBCURL - #include -#endif #include #include #include @@ -16,9 +13,6 @@ #include #include -#if !CLP_S_EXCLUDE_LIBCURL - #include -#endif #include #include #include @@ -32,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -42,6 +35,7 @@ #include #include #include +#include using clp::ffi::ir_stream::Deserializer; using clp::ffi::ir_stream::IRErrorCode; @@ -677,7 +671,7 @@ bool JsonParser::ingest() { case FileType::Zstd: case FileType::Unknown: default: { - std::ignore = check_and_log_curl_error(path, reader); + std::ignore = NetworkUtils::check_and_log_curl_error(path.path, reader.get()); SPDLOG_ERROR("Could not deduce content type for input {}", path.path); std::ignore = m_archive_writer->close(); return false; @@ -685,7 +679,9 @@ bool JsonParser::ingest() { } close_nested_readers(nested_readers); - if (false == ingestion_successful || check_and_log_curl_error(path, reader)) { + if (false == ingestion_successful + || NetworkUtils::check_and_log_curl_error(path.path, reader.get())) + { std::ignore = m_archive_writer->close(); return false; } @@ -1356,41 +1352,4 @@ void JsonParser::split_archive() { m_archive_options.id = m_generator(); m_archive_writer->open(m_archive_options); } - -#if CLP_S_EXCLUDE_LIBCURL -bool JsonParser::check_and_log_curl_error( - Path const& path, - std::shared_ptr reader -) { - if (auto network_reader = std::dynamic_pointer_cast(reader); - nullptr != network_reader) - { - throw std::runtime_error("Simplified static clp-s executable does not support libcurl."); - } - return false; -} -#else -bool JsonParser::check_and_log_curl_error( - Path const& path, - std::shared_ptr reader -) { - if (auto network_reader = std::dynamic_pointer_cast(reader); - nullptr != network_reader) - { - if (auto const rc = network_reader->get_curl_ret_code(); - rc.has_value() && CURLcode::CURLE_OK != rc.value()) - { - auto const curl_error_message = network_reader->get_curl_error_msg(); - SPDLOG_ERROR( - "Encountered curl error while ingesting {} - Code: {} - Message: {}", - path.path, - static_cast(rc.value()), - curl_error_message.value_or("Unknown error") - ); - return true; - } - } - return false; -} -#endif } // namespace clp_s diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index 82f8d04786..b3a3679d5f 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -211,16 +211,6 @@ class JsonParser { */ int32_t add_metadata_field(std::string_view const field_name, NodeType type); - /** - * Checks if a reader interface is a clp::NetworkReader that has encountered a CURL error and - * logs relevant CURL error information if a CURL error has occurred. - * @param path - * @param reader - * @return true if the provided ReaderInterface has experienced a CURL error and false otherwise - */ - static bool - check_and_log_curl_error(Path const& path, std::shared_ptr reader); - std::vector m_input_paths; NetworkAuthOption m_network_auth{}; diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 59b18a5bca..588cbba9ea 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -11,6 +11,9 @@ #include #include +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/NetworkReader.hpp" +#endif #include "archive_constants.hpp" using std::string; @@ -216,4 +219,31 @@ void StringUtils::escape_json_string(std::string& destination, std::string_view } append_unescaped_slice(source.size()); } + +auto +NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) + -> bool { +#if CLP_S_EXCLUDE_LIBCURL + std::ignore = path; + std::ignore = reader; + return false; +#else + auto const* network_reader = dynamic_cast(reader); + if (nullptr == network_reader) { + return false; + } + if (auto const curl_error_info = network_reader->get_curl_error_info(); + curl_error_info.has_value()) + { + SPDLOG_ERROR( + "Encountered curl error while downloading {} - Code: {} - Message: {}", + path, + curl_error_info->code, + curl_error_info->message + ); + return true; + } + return false; +#endif +} } // namespace clp_s diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index a711eb8ad6..a5c71e1e29 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -10,6 +10,8 @@ #include #include +#include "../clp/ReaderInterface.hpp" + namespace clp_s { class FileUtils { public: @@ -113,6 +115,19 @@ class StringUtils { } }; +class NetworkUtils { +public: + /** + * Checks if a reader is a clp::NetworkReader that has encountered a CURL error and logs + * relevant CURL error information if a CURL error has occurred. + * @param path + * @param reader + * @return true if a curl error has occurred on the reader, false otherwise. + */ + [[nodiscard]] static auto + check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) -> bool; +}; + enum EvaluatedValue { True, False, diff --git a/components/core/src/clp_s/log_converter/log_converter.cpp b/components/core/src/clp_s/log_converter/log_converter.cpp index 108383b287..1984012e7a 100644 --- a/components/core/src/clp_s/log_converter/log_converter.cpp +++ b/components/core/src/clp_s/log_converter/log_converter.cpp @@ -5,13 +5,12 @@ #include #include -#include #include #include -#include "../../clp/NetworkReader.hpp" #include "../../clp/ReaderInterface.hpp" #include "../InputConfig.hpp" +#include "../Utils.hpp" #include "CommandLineArguments.hpp" #include "LogConverter.hpp" @@ -19,15 +18,6 @@ using clp_s::log_converter::CommandLineArguments; using clp_s::log_converter::LogConverter; namespace { -/** - * Checks for and logs CURL errors on a reader. - * @param path The path that the reader has opened. - * @param reader The open reader which may have experienced a CURL error. - * @return Whether a CURL error has occurred on the reader. - */ -[[nodiscard]] auto -check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* reader) -> bool; - /** * Converts all files according to the command line arguments. * @param command_line_arguments @@ -35,26 +25,6 @@ check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* re */ [[nodiscard]] auto convert_files(CommandLineArguments const& command_line_arguments) -> bool; -auto check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* reader) -> bool { - auto const* network_reader{dynamic_cast(reader)}; - if (nullptr == network_reader) { - return false; - } - if (auto const rc = network_reader->get_curl_ret_code(); - rc.has_value() && CURLcode::CURLE_OK != rc.value()) - { - auto const curl_error_message = network_reader->get_curl_error_msg(); - SPDLOG_ERROR( - "Encountered curl error while converting {} - Code: {} - Message: {}", - path.path, - static_cast(rc.value()), - curl_error_message.value_or("Unknown error.") - ); - return true; - } - return false; -} - auto convert_files(CommandLineArguments const& command_line_arguments) -> bool { LogConverter log_converter; @@ -86,7 +56,8 @@ auto convert_files(CommandLineArguments const& command_line_arguments) -> bool { case clp_s::FileType::Zstd: case clp_s::FileType::Unknown: default: { - std::ignore = check_and_log_curl_error(path, reader.get()); + std::ignore + = clp_s::NetworkUtils::check_and_log_curl_error(path.path, reader.get()); SPDLOG_ERROR("Received input that was not unstructured logtext: {}.", path.path); return false; } diff --git a/components/core/tests/test-NetworkReader.cpp b/components/core/tests/test-NetworkReader.cpp index 1f125b5371..891614a79b 100644 --- a/components/core/tests/test-NetworkReader.cpp +++ b/components/core/tests/test-NetworkReader.cpp @@ -96,9 +96,9 @@ auto assert_curl_error_code(CURLcode expected, clp::NetworkReader const& reader) "Unexpected CURL error code: " + std::to_string(actual) + "; expected: " + std::to_string(expected) }; - auto const curl_error_message{reader.get_curl_error_msg()}; - if (curl_error_message.has_value()) { - message_to_log += "\nError message:\n" + std::string{curl_error_message.value()}; + auto const curl_error_info{reader.get_curl_error_info()}; + if (curl_error_info.has_value()) { + message_to_log += "\nError message:\n" + std::string{curl_error_info->message}; } WARN(message_to_log); return false; From 4b5729e853657b3ee3d985e50c0f2bb3e8af44dc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 15:50:01 -0500 Subject: [PATCH 26/31] Improve clp-s cmakelist --- components/core/src/clp_s/CMakeLists.txt | 45 +++++++++--------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index d24be8efaf..7b19ff53b6 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -19,13 +19,6 @@ set( ../clp/BufferedReader.hpp ../clp/BufferReader.cpp ../clp/BufferReader.hpp - ../clp/CurlDownloadHandler.cpp - ../clp/CurlDownloadHandler.hpp - ../clp/CurlEasyHandle.hpp - ../clp/CurlGlobalInstance.cpp - ../clp/CurlGlobalInstance.hpp - ../clp/CurlOperationFailed.hpp - ../clp/CurlStringList.hpp ../clp/cli_utils.cpp ../clp/cli_utils.hpp ../clp/Defs.h @@ -92,8 +85,6 @@ set( ../clp/LogSurgeonReader.hpp ../clp/LogTypeDictionaryEntryReq.hpp ../clp/LogTypeDictionaryReaderReq.hpp - ../clp/NetworkReader.cpp - ../clp/NetworkReader.hpp ../clp/networking/socket_utils.cpp ../clp/networking/socket_utils.hpp ../clp/Query.cpp @@ -128,6 +119,22 @@ set( ../clp/WriterInterface.hpp ) +set( + CLP_S_CLP_CURL_SOURCES + ../clp/CurlDownloadHandler.cpp + ../clp/CurlDownloadHandler.hpp + ../clp/CurlEasyHandle.hpp + ../clp/CurlGlobalInstance.cpp + ../clp/CurlGlobalInstance.hpp + ../clp/CurlOperationFailed.hpp + ../clp/CurlStringList.hpp + ../clp/NetworkReader.cpp + ../clp/NetworkReader.hpp +) +if(NOT CLP_S_EXCLUDE_LIBCURL) + list(APPEND CLP_S_CLP_SOURCES ${CLP_S_CLP_CURL_SOURCES}) +endif() + # This library is intended as a temporary stand-in until clp has been packaged into libraries. if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) add_library( @@ -149,6 +156,7 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) fmt::fmt msgpack-cxx nlohmann_json::nlohmann_json + OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling ) @@ -159,11 +167,6 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) ${CURL_LIBRARIES} ) endif() - target_link_libraries( - clp_s_clp_dependencies - PRIVATE - OpenSSL::Crypto - ) endif() set( @@ -324,13 +327,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER) fmt::fmt spdlog::spdlog ) - if(NOT CLP_S_EXCLUDE_LIBCURL) - target_link_libraries( - clp_s_archive_writer - PRIVATE - ${CURL_LIBRARIES} - ) - endif() endif() set( @@ -400,13 +396,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) fmt::fmt spdlog::spdlog ) - if(NOT CLP_S_EXCLUDE_LIBCURL) - target_link_libraries( - clp_s_archive_reader - PRIVATE - ${CURL_LIBRARIES} - ) - endif() endif() set( From 37a3b5d51f5952bcdfa883fdb4ff65388747e7d7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 16:22:06 -0500 Subject: [PATCH 27/31] polish --- components/core/src/clp_s/CMakeLists.txt | 12 ++---------- components/core/src/clp_s/Utils.cpp | 2 +- components/core/src/clp_s/Utils.hpp | 6 +++--- components/core/src/clp_s/clp-s.cpp | 1 - 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 7b19ff53b6..a0b1678036 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -421,12 +421,8 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) ${zstd_TARGET} PRIVATE fmt::fmt - spdlog::spdlog - ) - target_link_libraries( - clp_s_json_constructor - PRIVATE ${MONGOCXX_TARGET} + spdlog::spdlog ) endif() @@ -494,15 +490,11 @@ if(CLP_BUILD_EXECUTABLES) clp_s::search clp_s::search::ast clp_s::search::kql + ${MONGOCXX_TARGET} msgpack-cxx spdlog::spdlog ystdlib::error_handling ) - target_link_libraries( - clp-s - PRIVATE - ${MONGOCXX_TARGET} - ) set_target_properties( clp-s PROPERTIES diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 588cbba9ea..077c1a2156 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -236,7 +236,7 @@ NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterfa curl_error_info.has_value()) { SPDLOG_ERROR( - "Encountered curl error while downloading {} - Code: {} - Message: {}", + "Encountered curl error while reading {} - Code: {} - Message: {}", path, curl_error_info->code, curl_error_info->message diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index a5c71e1e29..5b949dc293 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -120,9 +120,9 @@ class NetworkUtils { /** * Checks if a reader is a clp::NetworkReader that has encountered a CURL error and logs * relevant CURL error information if a CURL error has occurred. - * @param path - * @param reader - * @return true if a curl error has occurred on the reader, false otherwise. + * @param path The path that the reader has opened. + * @param reader The open reader which may have experienced a CURL error. + * @return true if a CURL error has occurred on the reader, false otherwise. */ [[nodiscard]] static auto check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) -> bool; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index a9f7c97d05..a54f999b86 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include From 12a543c6df77bbdb88f2935a10f8a9aa2043bc49 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 16:54:39 -0500 Subject: [PATCH 28/31] Use CURLcode for curl error structure --- components/core/src/clp/NetworkReader.hpp | 4 ++-- components/core/src/clp_s/Utils.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/NetworkReader.hpp b/components/core/src/clp/NetworkReader.hpp index 1a436ae054..76acb8dfa7 100644 --- a/components/core/src/clp/NetworkReader.hpp +++ b/components/core/src/clp/NetworkReader.hpp @@ -70,7 +70,7 @@ class NetworkReader : public ReaderInterface { * CURL error details set by the underlying CURL handler. */ struct CurlErrorInfo { - int64_t code; + CURLcode code; std::string_view message; }; @@ -241,7 +241,7 @@ class NetworkReader : public ReaderInterface { return std::nullopt; } return CurlErrorInfo{ - .code = static_cast(ret_code.value()), + .code = ret_code.value(), .message = std::string_view{m_curl_error_msg_buf->data()} }; } diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 077c1a2156..0b037b629a 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -238,7 +238,7 @@ NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterfa SPDLOG_ERROR( "Encountered curl error while reading {} - Code: {} - Message: {}", path, - curl_error_info->code, + static_cast(curl_error_info->code), curl_error_info->message ); return true; From 5731db2a79c96f3e2cb4f280a168e403e51c9c72 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 17:35:52 -0500 Subject: [PATCH 29/31] Use whole-function conditional and disable openssl::crypto --- components/core/src/clp_s/CMakeLists.txt | 10 ++-- components/core/src/clp_s/InputConfig.cpp | 26 +++++----- components/core/src/clp_s/Utils.cpp | 60 +++++++++++++---------- components/core/src/clp_s/Utils.hpp | 26 +++++----- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index a0b1678036..65669c554d 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -11,8 +11,6 @@ add_subdirectory(timestamp_parser) set( CLP_S_CLP_SOURCES - ../clp/aws/AwsAuthenticationSigner.cpp - ../clp/aws/AwsAuthenticationSigner.hpp ../clp/BoundedReader.cpp ../clp/BoundedReader.hpp ../clp/BufferedReader.cpp @@ -71,8 +69,6 @@ set( ../clp/FileReader.hpp ../clp/GrepCore.cpp ../clp/GrepCore.hpp - ../clp/hash_utils.cpp - ../clp/hash_utils.hpp ../clp/SchemaSearcher.cpp ../clp/SchemaSearcher.hpp ../clp/ir/constants.hpp @@ -121,6 +117,10 @@ set( set( CLP_S_CLP_CURL_SOURCES + ../clp/aws/AwsAuthenticationSigner.cpp + ../clp/aws/AwsAuthenticationSigner.hpp + ../clp/hash_utils.cpp + ../clp/hash_utils.hpp ../clp/CurlDownloadHandler.cpp ../clp/CurlDownloadHandler.hpp ../clp/CurlEasyHandle.hpp @@ -156,7 +156,6 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) fmt::fmt msgpack-cxx nlohmann_json::nlohmann_json - OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling ) @@ -165,6 +164,7 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) clp_s_clp_dependencies PRIVATE ${CURL_LIBRARIES} + OpenSSL::Crypto ) endif() endif() diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index d0e5dca971..2be98defc1 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -8,20 +8,21 @@ #include #include #include -#if CLP_S_EXCLUDE_LIBCURL - #include -#endif #include #include #include #include -#include "../clp/aws/AwsAuthenticationSigner.hpp" +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/aws/AwsAuthenticationSigner.hpp" +#endif #include "../clp/BufferedReader.hpp" #include "../clp/ffi/ir_stream/protocol_constants.hpp" #include "../clp/FileReader.hpp" -#include "../clp/NetworkReader.hpp" +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/NetworkReader.hpp" +#endif #include "../clp/ReaderInterface.hpp" #include "../clp/spdlog_with_specializations.hpp" #include "../clp/streaming_compression/Decompressor.hpp" @@ -158,6 +159,15 @@ auto try_create_file_reader(std::string_view const file_path) } } +#if CLP_S_EXCLUDE_LIBCURL +auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) + -> std::shared_ptr { + std::ignore = url; + std::ignore = auth; + SPDLOG_ERROR("This build of clp-s does not support network inputs (libcurl excluded)."); + return nullptr; +} +#else auto try_sign_url(std::string& url) -> bool { auto const aws_access_key = std::getenv(cAwsAccessKeyIdEnvVar); auto const aws_secret_access_key = std::getenv(cAwsSecretAccessKeyEnvVar); @@ -194,12 +204,6 @@ auto try_sign_url(std::string& url) -> bool { return true; } -#if CLP_S_EXCLUDE_LIBCURL -auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) - -> std::shared_ptr { - throw std::runtime_error("Simplified static clp-s executable does not support libcurl."); -} -#else auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) -> std::shared_ptr { std::string request_url{url}; diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 0b037b629a..ef05a58cf3 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -152,6 +152,39 @@ bool FileUtils::get_last_non_empty_path_component(std::string_view const path, s return false; } +#if CLP_S_EXCLUDE_LIBCURL +auto NetworkUtils::check_and_log_curl_error( + std::string_view path, + clp::ReaderInterface const* reader +) -> bool { + std::ignore = path; + std::ignore = reader; + return false; +} +#else +auto NetworkUtils::check_and_log_curl_error( + std::string_view path, + clp::ReaderInterface const* reader +) -> bool { + auto const* network_reader = dynamic_cast(reader); + if (nullptr == network_reader) { + return false; + } + if (auto const curl_error_info = network_reader->get_curl_error_info(); + curl_error_info.has_value()) + { + SPDLOG_ERROR( + "Encountered curl error while reading {} - Code: {} - Message: {}", + path, + static_cast(curl_error_info->code), + curl_error_info->message + ); + return true; + } + return false; +} +#endif + bool UriUtils::get_last_uri_component(std::string_view const uri, std::string& name) { auto parsed_result = boost::urls::parse_uri(uri); if (false == parsed_result.has_value()) { @@ -219,31 +252,4 @@ void StringUtils::escape_json_string(std::string& destination, std::string_view } append_unescaped_slice(source.size()); } - -auto -NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) - -> bool { -#if CLP_S_EXCLUDE_LIBCURL - std::ignore = path; - std::ignore = reader; - return false; -#else - auto const* network_reader = dynamic_cast(reader); - if (nullptr == network_reader) { - return false; - } - if (auto const curl_error_info = network_reader->get_curl_error_info(); - curl_error_info.has_value()) - { - SPDLOG_ERROR( - "Encountered curl error while reading {} - Code: {} - Message: {}", - path, - static_cast(curl_error_info->code), - curl_error_info->message - ); - return true; - } - return false; -#endif -} } // namespace clp_s diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index 5b949dc293..841ae72f19 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -49,6 +49,19 @@ class FileUtils { static bool get_last_non_empty_path_component(std::string_view const path, std::string& name); }; +class NetworkUtils { +public: + /** + * Checks if a reader is a clp::NetworkReader that has encountered a CURL error and logs + * relevant CURL error information if a CURL error has occurred. + * @param path The path that the reader has opened. + * @param reader The open reader which may have experienced a CURL error. + * @return true if a CURL error has occurred on the reader, false otherwise. + */ + [[nodiscard]] static auto + check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) -> bool; +}; + class UriUtils { public: /** @@ -115,19 +128,6 @@ class StringUtils { } }; -class NetworkUtils { -public: - /** - * Checks if a reader is a clp::NetworkReader that has encountered a CURL error and logs - * relevant CURL error information if a CURL error has occurred. - * @param path The path that the reader has opened. - * @param reader The open reader which may have experienced a CURL error. - * @return true if a CURL error has occurred on the reader, false otherwise. - */ - [[nodiscard]] static auto - check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) -> bool; -}; - enum EvaluatedValue { True, False, From 260548fe4cb2476aaf44033c4c3ab071627e9edd Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 17:45:50 -0500 Subject: [PATCH 30/31] Minor polish --- components/core/src/clp_s/CMakeLists.txt | 10 +++++----- components/core/src/clp_s/InputConfig.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 65669c554d..a59942e87d 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -1,14 +1,14 @@ +add_subdirectory(indexer) +add_subdirectory(log_converter) +add_subdirectory(search) +add_subdirectory(timestamp_parser) + option( CLP_S_EXCLUDE_LIBCURL "Exclude libcurl support from clp-s." ON ) -add_subdirectory(indexer) -add_subdirectory(log_converter) -add_subdirectory(search) -add_subdirectory(timestamp_parser) - set( CLP_S_CLP_SOURCES ../clp/BoundedReader.cpp diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index 2be98defc1..1b5b1473ea 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -164,7 +164,7 @@ auto try_create_network_reader(std::string_view const url, NetworkAuthOption con -> std::shared_ptr { std::ignore = url; std::ignore = auth; - SPDLOG_ERROR("This build of clp-s does not support network inputs (libcurl excluded)."); + SPDLOG_ERROR("This build of clp-s does not support network reading (libcurl excluded)."); return nullptr; } #else From 56e27d43241172e5c01ae60dc121d1df2a51067d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 27 Feb 2026 17:54:13 -0500 Subject: [PATCH 31/31] lint fix --- components/core/src/clp_s/Utils.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index ef05a58cf3..f73513df44 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -153,19 +153,17 @@ bool FileUtils::get_last_non_empty_path_component(std::string_view const path, s } #if CLP_S_EXCLUDE_LIBCURL -auto NetworkUtils::check_and_log_curl_error( - std::string_view path, - clp::ReaderInterface const* reader -) -> bool { +auto +NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) + -> bool { std::ignore = path; std::ignore = reader; return false; } #else -auto NetworkUtils::check_and_log_curl_error( - std::string_view path, - clp::ReaderInterface const* reader -) -> bool { +auto +NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) + -> bool { auto const* network_reader = dynamic_cast(reader); if (nullptr == network_reader) { return false;