diff --git a/components/core/src/clp/NetworkReader.hpp b/components/core/src/clp/NetworkReader.hpp index 37a9995c93..76acb8dfa7 100644 --- a/components/core/src/clp/NetworkReader.hpp +++ b/components/core/src/clp/NetworkReader.hpp @@ -66,6 +66,14 @@ class NetworkReader : public ReaderInterface { Finished }; + /** + * CURL error details set by the underlying CURL handler. + */ + struct CurlErrorInfo { + CURLcode code; + std::string_view message; + }; + // Constants static constexpr size_t cDefaultBufferPoolSize{8}; static constexpr size_t cDefaultBufferSize{4096}; @@ -224,16 +232,18 @@ class NetworkReader : public ReaderInterface { } /** - * @return The error message set by the underlying CURL handler. - * @return std::nullopt if the download is still in-progress or no error has occured. + * @return CURL error info if the download has completed with a CURL error. + * @return std::nullopt if the download is still in-progress or no error has occurred. */ - [[nodiscard]] auto get_curl_error_msg() const -> std::optional { - if (auto const ret_code{get_curl_ret_code()}; - false == ret_code.has_value() || CURLE_OK == ret_code.value()) - { + [[nodiscard]] auto get_curl_error_info() const -> std::optional { + auto const ret_code = get_curl_ret_code(); + if (false == ret_code.has_value() || CURLcode::CURLE_OK == ret_code.value()) { return std::nullopt; } - return std::string_view{m_curl_error_msg_buf->data()}; + return CurlErrorInfo{ + .code = ret_code.value(), + .message = std::string_view{m_curl_error_msg_buf->data()} + }; } private: diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index e0d9a126e2..bb2c9f675b 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -3,23 +3,20 @@ add_subdirectory(log_converter) add_subdirectory(search) add_subdirectory(timestamp_parser) +option( + CLP_S_EXCLUDE_LIBCURL + "Exclude libcurl support from clp-s." + ON +) + set( CLP_S_CLP_SOURCES - ../clp/aws/AwsAuthenticationSigner.cpp - ../clp/aws/AwsAuthenticationSigner.hpp ../clp/BoundedReader.cpp ../clp/BoundedReader.hpp ../clp/BufferedReader.cpp ../clp/BufferedReader.hpp ../clp/BufferReader.cpp ../clp/BufferReader.hpp - ../clp/CurlDownloadHandler.cpp - ../clp/CurlDownloadHandler.hpp - ../clp/CurlEasyHandle.hpp - ../clp/CurlGlobalInstance.cpp - ../clp/CurlGlobalInstance.hpp - ../clp/CurlOperationFailed.hpp - ../clp/CurlStringList.hpp ../clp/cli_utils.cpp ../clp/cli_utils.hpp ../clp/Defs.h @@ -72,8 +69,6 @@ set( ../clp/FileReader.hpp ../clp/GrepCore.cpp ../clp/GrepCore.hpp - ../clp/hash_utils.cpp - ../clp/hash_utils.hpp ../clp/SchemaSearcher.cpp ../clp/SchemaSearcher.hpp ../clp/ir/constants.hpp @@ -86,8 +81,6 @@ set( ../clp/LogSurgeonReader.hpp ../clp/LogTypeDictionaryEntryReq.hpp ../clp/LogTypeDictionaryReaderReq.hpp - ../clp/NetworkReader.cpp - ../clp/NetworkReader.hpp ../clp/networking/socket_utils.cpp ../clp/networking/socket_utils.hpp ../clp/Query.cpp @@ -122,6 +115,26 @@ set( ../clp/WriterInterface.hpp ) +set( + CLP_S_CLP_CURL_SOURCES + ../clp/aws/AwsAuthenticationSigner.cpp + ../clp/aws/AwsAuthenticationSigner.hpp + ../clp/hash_utils.cpp + ../clp/hash_utils.hpp + ../clp/CurlDownloadHandler.cpp + ../clp/CurlDownloadHandler.hpp + ../clp/CurlEasyHandle.hpp + ../clp/CurlGlobalInstance.cpp + ../clp/CurlGlobalInstance.hpp + ../clp/CurlOperationFailed.hpp + ../clp/CurlStringList.hpp + ../clp/NetworkReader.cpp + ../clp/NetworkReader.hpp +) +if(NOT CLP_S_EXCLUDE_LIBCURL) + list(APPEND CLP_S_CLP_SOURCES ${CLP_S_CLP_CURL_SOURCES}) +endif() + # This library is intended as a temporary stand-in until clp has been packaged into libraries. if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) add_library( @@ -140,14 +153,20 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) zstd::libzstd_static PRIVATE Boost::regex - ${CURL_LIBRARIES} fmt::fmt msgpack-cxx nlohmann_json::nlohmann_json - OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling ) + if(NOT CLP_S_EXCLUDE_LIBCURL) + target_link_libraries( + clp_s_clp_dependencies + PRIVATE + ${CURL_LIBRARIES} + OpenSSL::Crypto + ) + endif() endif() set( @@ -218,6 +237,11 @@ if(CLP_BUILD_CLP_S_IO) ${CLP_S_IO_SOURCES} ) add_library(clp_s::io ALIAS clp_s_io) + target_compile_definitions( + clp_s_io + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp_s_io PRIVATE cxx_std_20) target_include_directories(clp_s_io PUBLIC ../) target_link_libraries( @@ -280,6 +304,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER) ${CLP_S_ARCHIVE_WRITER_SOURCES} ) add_library(clp_s::archive_writer ALIAS clp_s_archive_writer) + target_compile_definitions( + clp_s_archive_writer + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp_s_archive_writer PRIVATE cxx_std_20) target_include_directories(clp_s_archive_writer PUBLIC ../) target_link_libraries( @@ -296,7 +325,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER) ystdlib::error_handling PRIVATE Boost::url - ${CURL_LIBRARIES} fmt::fmt spdlog::spdlog ) @@ -345,6 +373,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) ${CLP_S_ARCHIVE_READER_SOURCES} ) add_library(clp_s::archive_reader ALIAS clp_s_archive_reader) + target_compile_definitions( + clp_s_archive_reader + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp_s_archive_reader PRIVATE cxx_std_20) target_include_directories(clp_s_archive_reader PUBLIC ../) target_link_libraries( @@ -361,7 +394,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER) PRIVATE Boost::url clp_s::clp_dependencies - ${CURL_LIBRARIES} fmt::fmt spdlog::spdlog ) @@ -438,6 +470,11 @@ if(CLP_BUILD_EXECUTABLES) clp-s.cpp ${CLP_S_EXE_SOURCES} ) + target_compile_definitions( + clp-s + PRIVATE + CLP_S_EXCLUDE_LIBCURL=$ + ) target_compile_features(clp-s PRIVATE cxx_std_20) target_include_directories(clp-s PRIVATE ../) target_link_libraries( diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index fd5b38dafa..1b5b1473ea 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -14,11 +14,15 @@ #include #include -#include "../clp/aws/AwsAuthenticationSigner.hpp" +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/aws/AwsAuthenticationSigner.hpp" +#endif #include "../clp/BufferedReader.hpp" #include "../clp/ffi/ir_stream/protocol_constants.hpp" #include "../clp/FileReader.hpp" -#include "../clp/NetworkReader.hpp" +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/NetworkReader.hpp" +#endif #include "../clp/ReaderInterface.hpp" #include "../clp/spdlog_with_specializations.hpp" #include "../clp/streaming_compression/Decompressor.hpp" @@ -155,6 +159,15 @@ auto try_create_file_reader(std::string_view const file_path) } } +#if CLP_S_EXCLUDE_LIBCURL +auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) + -> std::shared_ptr { + std::ignore = url; + std::ignore = auth; + SPDLOG_ERROR("This build of clp-s does not support network reading (libcurl excluded)."); + return nullptr; +} +#else auto try_sign_url(std::string& url) -> bool { auto const aws_access_key = std::getenv(cAwsAccessKeyIdEnvVar); auto const aws_secret_access_key = std::getenv(cAwsSecretAccessKeyEnvVar); @@ -213,6 +226,7 @@ auto try_create_network_reader(std::string_view const url, NetworkAuthOption con return nullptr; } } +#endif auto could_be_zstd(char const* peek_buf, size_t peek_size) -> bool { constexpr std::array cZstdMagicNumber = {'\x28', '\xB5', '\x2F', '\xFD'}; diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 7115a4ea53..3adc8deb01 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -13,7 +13,6 @@ #include #include -#include #include #include #include @@ -27,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -37,6 +35,7 @@ #include #include #include +#include using clp::ffi::ir_stream::Deserializer; using clp::ffi::ir_stream::IRErrorCode; @@ -672,7 +671,7 @@ bool JsonParser::ingest() { case FileType::Zstd: case FileType::Unknown: default: { - std::ignore = check_and_log_curl_error(path, reader); + std::ignore = NetworkUtils::check_and_log_curl_error(path.path, reader.get()); SPDLOG_ERROR("Could not deduce content type for input {}", path.path); std::ignore = m_archive_writer->close(); return false; @@ -680,7 +679,9 @@ bool JsonParser::ingest() { } close_nested_readers(nested_readers); - if (false == ingestion_successful || check_and_log_curl_error(path, reader)) { + if (false == ingestion_successful + || NetworkUtils::check_and_log_curl_error(path.path, reader.get())) + { std::ignore = m_archive_writer->close(); return false; } @@ -1351,27 +1352,4 @@ void JsonParser::split_archive() { m_archive_options.id = m_generator(); m_archive_writer->open(m_archive_options); } - -bool JsonParser::check_and_log_curl_error( - Path const& path, - std::shared_ptr reader -) { - if (auto network_reader = std::dynamic_pointer_cast(reader); - nullptr != network_reader) - { - if (auto const rc = network_reader->get_curl_ret_code(); - rc.has_value() && CURLcode::CURLE_OK != rc.value()) - { - auto const curl_error_message = network_reader->get_curl_error_msg(); - SPDLOG_ERROR( - "Encountered curl error while ingesting {} - Code: {} - Message: {}", - path.path, - static_cast(rc.value()), - curl_error_message.value_or("Unknown error") - ); - return true; - } - } - return false; -} } // namespace clp_s diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index 82f8d04786..b3a3679d5f 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -211,16 +211,6 @@ class JsonParser { */ int32_t add_metadata_field(std::string_view const field_name, NodeType type); - /** - * Checks if a reader interface is a clp::NetworkReader that has encountered a CURL error and - * logs relevant CURL error information if a CURL error has occurred. - * @param path - * @param reader - * @return true if the provided ReaderInterface has experienced a CURL error and false otherwise - */ - static bool - check_and_log_curl_error(Path const& path, std::shared_ptr reader); - std::vector m_input_paths; NetworkAuthOption m_network_auth{}; diff --git a/components/core/src/clp_s/Utils.cpp b/components/core/src/clp_s/Utils.cpp index 59b18a5bca..f73513df44 100644 --- a/components/core/src/clp_s/Utils.cpp +++ b/components/core/src/clp_s/Utils.cpp @@ -11,6 +11,9 @@ #include #include +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/NetworkReader.hpp" +#endif #include "archive_constants.hpp" using std::string; @@ -149,6 +152,37 @@ bool FileUtils::get_last_non_empty_path_component(std::string_view const path, s return false; } +#if CLP_S_EXCLUDE_LIBCURL +auto +NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) + -> bool { + std::ignore = path; + std::ignore = reader; + return false; +} +#else +auto +NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) + -> bool { + auto const* network_reader = dynamic_cast(reader); + if (nullptr == network_reader) { + return false; + } + if (auto const curl_error_info = network_reader->get_curl_error_info(); + curl_error_info.has_value()) + { + SPDLOG_ERROR( + "Encountered curl error while reading {} - Code: {} - Message: {}", + path, + static_cast(curl_error_info->code), + curl_error_info->message + ); + return true; + } + return false; +} +#endif + bool UriUtils::get_last_uri_component(std::string_view const uri, std::string& name) { auto parsed_result = boost::urls::parse_uri(uri); if (false == parsed_result.has_value()) { diff --git a/components/core/src/clp_s/Utils.hpp b/components/core/src/clp_s/Utils.hpp index a711eb8ad6..841ae72f19 100644 --- a/components/core/src/clp_s/Utils.hpp +++ b/components/core/src/clp_s/Utils.hpp @@ -10,6 +10,8 @@ #include #include +#include "../clp/ReaderInterface.hpp" + namespace clp_s { class FileUtils { public: @@ -47,6 +49,19 @@ class FileUtils { static bool get_last_non_empty_path_component(std::string_view const path, std::string& name); }; +class NetworkUtils { +public: + /** + * Checks if a reader is a clp::NetworkReader that has encountered a CURL error and logs + * relevant CURL error information if a CURL error has occurred. + * @param path The path that the reader has opened. + * @param reader The open reader which may have experienced a CURL error. + * @return true if a CURL error has occurred on the reader, false otherwise. + */ + [[nodiscard]] static auto + check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader) -> bool; +}; + class UriUtils { public: /** diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 87d2b0af2b..a54f999b86 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -13,7 +13,9 @@ #include #include -#include "../clp/CurlGlobalInstance.hpp" +#if !CLP_S_EXCLUDE_LIBCURL + #include "../clp/CurlGlobalInstance.hpp" +#endif #include "../clp/ir/constants.hpp" #include "../clp/streaming_archive/ArchiveMetadata.hpp" #include "../reducer/network_utils.hpp" @@ -305,7 +307,9 @@ int main(int argc, char const* argv[]) { } mongocxx::instance const mongocxx_instance{}; +#if !CLP_S_EXCLUDE_LIBCURL clp::CurlGlobalInstance const curl_instance{}; +#endif CommandLineArguments command_line_arguments("clp-s"); auto parsing_result = command_line_arguments.parse_arguments(argc, argv); diff --git a/components/core/src/clp_s/log_converter/log_converter.cpp b/components/core/src/clp_s/log_converter/log_converter.cpp index 108383b287..1984012e7a 100644 --- a/components/core/src/clp_s/log_converter/log_converter.cpp +++ b/components/core/src/clp_s/log_converter/log_converter.cpp @@ -5,13 +5,12 @@ #include #include -#include #include #include -#include "../../clp/NetworkReader.hpp" #include "../../clp/ReaderInterface.hpp" #include "../InputConfig.hpp" +#include "../Utils.hpp" #include "CommandLineArguments.hpp" #include "LogConverter.hpp" @@ -19,15 +18,6 @@ using clp_s::log_converter::CommandLineArguments; using clp_s::log_converter::LogConverter; namespace { -/** - * Checks for and logs CURL errors on a reader. - * @param path The path that the reader has opened. - * @param reader The open reader which may have experienced a CURL error. - * @return Whether a CURL error has occurred on the reader. - */ -[[nodiscard]] auto -check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* reader) -> bool; - /** * Converts all files according to the command line arguments. * @param command_line_arguments @@ -35,26 +25,6 @@ check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* re */ [[nodiscard]] auto convert_files(CommandLineArguments const& command_line_arguments) -> bool; -auto check_and_log_curl_error(clp_s::Path const& path, clp::ReaderInterface const* reader) -> bool { - auto const* network_reader{dynamic_cast(reader)}; - if (nullptr == network_reader) { - return false; - } - if (auto const rc = network_reader->get_curl_ret_code(); - rc.has_value() && CURLcode::CURLE_OK != rc.value()) - { - auto const curl_error_message = network_reader->get_curl_error_msg(); - SPDLOG_ERROR( - "Encountered curl error while converting {} - Code: {} - Message: {}", - path.path, - static_cast(rc.value()), - curl_error_message.value_or("Unknown error.") - ); - return true; - } - return false; -} - auto convert_files(CommandLineArguments const& command_line_arguments) -> bool { LogConverter log_converter; @@ -86,7 +56,8 @@ auto convert_files(CommandLineArguments const& command_line_arguments) -> bool { case clp_s::FileType::Zstd: case clp_s::FileType::Unknown: default: { - std::ignore = check_and_log_curl_error(path, reader.get()); + std::ignore + = clp_s::NetworkUtils::check_and_log_curl_error(path.path, reader.get()); SPDLOG_ERROR("Received input that was not unstructured logtext: {}.", path.path); return false; } diff --git a/components/core/tests/test-NetworkReader.cpp b/components/core/tests/test-NetworkReader.cpp index 1f125b5371..891614a79b 100644 --- a/components/core/tests/test-NetworkReader.cpp +++ b/components/core/tests/test-NetworkReader.cpp @@ -96,9 +96,9 @@ auto assert_curl_error_code(CURLcode expected, clp::NetworkReader const& reader) "Unexpected CURL error code: " + std::to_string(actual) + "; expected: " + std::to_string(expected) }; - auto const curl_error_message{reader.get_curl_error_msg()}; - if (curl_error_message.has_value()) { - message_to_log += "\nError message:\n" + std::string{curl_error_message.value()}; + auto const curl_error_info{reader.get_curl_error_info()}; + if (curl_error_info.has_value()) { + message_to_log += "\nError message:\n" + std::string{curl_error_info->message}; } WARN(message_to_log); return false;