feat(clp-s): Add CLP_S_DISABLE_CURL build option to exclude libcurl/libcrypto paths for WebAssembly compatibility; Improve Curl error handling utilities.

components/core/src/clp/NetworkReader.hpp

@@ -66,6 +66,14 @@ class NetworkReader : public ReaderInterface {
         Finished
     };
 
+    /**
+     * CURL error details set by the underlying CURL handler.
+     */
+    struct CurlErrorInfo {
+        CURLcode code;
+        std::string_view message;
+    };
+
     // Constants
     static constexpr size_t cDefaultBufferPoolSize{8};
     static constexpr size_t cDefaultBufferSize{4096};
@@ -224,16 +232,18 @@ class NetworkReader : public ReaderInterface {
     }
 
     /**
-     * @return The error message set by the underlying CURL handler.
-     * @return std::nullopt if the download is still in-progress or no error has occured.
+     * @return CURL error info if the download has completed with a CURL error.
+     * @return std::nullopt if the download is still in-progress or no error has occurred.
      */
-    [[nodiscard]] auto get_curl_error_msg() const -> std::optional<std::string_view> {
-        if (auto const ret_code{get_curl_ret_code()};
-            false == ret_code.has_value() || CURLE_OK == ret_code.value())
-        {
+    [[nodiscard]] auto get_curl_error_info() const -> std::optional<CurlErrorInfo> {
+        auto const ret_code = get_curl_ret_code();
+        if (false == ret_code.has_value() || CURLcode::CURLE_OK == ret_code.value()) {
             return std::nullopt;
         }
-        return std::string_view{m_curl_error_msg_buf->data()};
+        return CurlErrorInfo{
+                .code = ret_code.value(),
+                .message = std::string_view{m_curl_error_msg_buf->data()}
+        };
     }
 
 private:

components/core/src/clp_s/CMakeLists.txt

@@ -3,23 +3,20 @@ add_subdirectory(log_converter)
 add_subdirectory(search)
 add_subdirectory(timestamp_parser)
 
+option(
+        CLP_S_EXCLUDE_LIBCURL
+        "Exclude libcurl support from clp-s."
+        ON
+)
+
 set(
         CLP_S_CLP_SOURCES
-        ../clp/aws/AwsAuthenticationSigner.cpp
-        ../clp/aws/AwsAuthenticationSigner.hpp
         ../clp/BoundedReader.cpp
         ../clp/BoundedReader.hpp
         ../clp/BufferedReader.cpp
         ../clp/BufferedReader.hpp
         ../clp/BufferReader.cpp
         ../clp/BufferReader.hpp
-        ../clp/CurlDownloadHandler.cpp
-        ../clp/CurlDownloadHandler.hpp
-        ../clp/CurlEasyHandle.hpp
-        ../clp/CurlGlobalInstance.cpp
-        ../clp/CurlGlobalInstance.hpp
-        ../clp/CurlOperationFailed.hpp
-        ../clp/CurlStringList.hpp
         ../clp/cli_utils.cpp
         ../clp/cli_utils.hpp
         ../clp/Defs.h
@@ -72,8 +69,6 @@ set(
         ../clp/FileReader.hpp
         ../clp/GrepCore.cpp
         ../clp/GrepCore.hpp
-        ../clp/hash_utils.cpp
-        ../clp/hash_utils.hpp
         ../clp/SchemaSearcher.cpp
         ../clp/SchemaSearcher.hpp
         ../clp/ir/constants.hpp
@@ -86,8 +81,6 @@ set(
         ../clp/LogSurgeonReader.hpp
         ../clp/LogTypeDictionaryEntryReq.hpp
         ../clp/LogTypeDictionaryReaderReq.hpp
-        ../clp/NetworkReader.cpp
-        ../clp/NetworkReader.hpp
         ../clp/networking/socket_utils.cpp
         ../clp/networking/socket_utils.hpp
         ../clp/Query.cpp
@@ -122,6 +115,26 @@ set(
         ../clp/WriterInterface.hpp
 )
 
+set(
+        CLP_S_CLP_CURL_SOURCES
+        ../clp/aws/AwsAuthenticationSigner.cpp
+        ../clp/aws/AwsAuthenticationSigner.hpp
+        ../clp/hash_utils.cpp
+        ../clp/hash_utils.hpp
+        ../clp/CurlDownloadHandler.cpp
+        ../clp/CurlDownloadHandler.hpp
+        ../clp/CurlEasyHandle.hpp
+        ../clp/CurlGlobalInstance.cpp
+        ../clp/CurlGlobalInstance.hpp
+        ../clp/CurlOperationFailed.hpp
+        ../clp/CurlStringList.hpp
+        ../clp/NetworkReader.cpp
+        ../clp/NetworkReader.hpp
+)
+if(NOT CLP_S_EXCLUDE_LIBCURL)
+        list(APPEND CLP_S_CLP_SOURCES ${CLP_S_CLP_CURL_SOURCES})
+endif()
+
 # This library is intended as a temporary stand-in until clp has been packaged into libraries.
 if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES)
         add_library(
@@ -140,14 +153,20 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES)
                 zstd::libzstd_static
                 PRIVATE
                 Boost::regex
-                ${CURL_LIBRARIES}
                 fmt::fmt
                 msgpack-cxx
                 nlohmann_json::nlohmann_json
-                OpenSSL::Crypto
                 spdlog::spdlog
                 ystdlib::error_handling
         )
+        if(NOT CLP_S_EXCLUDE_LIBCURL)
+                target_link_libraries(
+                        clp_s_clp_dependencies
+                        PRIVATE
+                        ${CURL_LIBRARIES}
+                        OpenSSL::Crypto
+                )
+        endif()
 endif()
 
 set(
@@ -218,6 +237,11 @@ if(CLP_BUILD_CLP_S_IO)
                 ${CLP_S_IO_SOURCES}
         )
         add_library(clp_s::io ALIAS clp_s_io)
+        target_compile_definitions(
+                clp_s_io
+                PRIVATE
+                CLP_S_EXCLUDE_LIBCURL=$<BOOL:${CLP_S_EXCLUDE_LIBCURL}>
+        )
         target_compile_features(clp_s_io PRIVATE cxx_std_20)
         target_include_directories(clp_s_io PUBLIC ../)
         target_link_libraries(
@@ -280,6 +304,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER)
                 ${CLP_S_ARCHIVE_WRITER_SOURCES}
         )
         add_library(clp_s::archive_writer ALIAS clp_s_archive_writer)
+        target_compile_definitions(
+                clp_s_archive_writer
+                PRIVATE
+                CLP_S_EXCLUDE_LIBCURL=$<BOOL:${CLP_S_EXCLUDE_LIBCURL}>
+        )
         target_compile_features(clp_s_archive_writer PRIVATE cxx_std_20)
         target_include_directories(clp_s_archive_writer PUBLIC ../)
         target_link_libraries(
@@ -296,7 +325,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEWRITER)
                 ystdlib::error_handling
                 PRIVATE
                 Boost::url
-                ${CURL_LIBRARIES}
                 fmt::fmt
                 spdlog::spdlog
         )
@@ -345,6 +373,11 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER)
                 ${CLP_S_ARCHIVE_READER_SOURCES}
         )
         add_library(clp_s::archive_reader ALIAS clp_s_archive_reader)
+        target_compile_definitions(
+                clp_s_archive_reader
+                PRIVATE
+                CLP_S_EXCLUDE_LIBCURL=$<BOOL:${CLP_S_EXCLUDE_LIBCURL}>
+        )
         target_compile_features(clp_s_archive_reader PRIVATE cxx_std_20)
         target_include_directories(clp_s_archive_reader PUBLIC ../)
         target_link_libraries(
@@ -361,7 +394,6 @@ if(CLP_BUILD_CLP_S_ARCHIVEREADER)
                 PRIVATE
                 Boost::url
                 clp_s::clp_dependencies
-                ${CURL_LIBRARIES}
                 fmt::fmt
                 spdlog::spdlog
         )
@@ -438,6 +470,11 @@ if(CLP_BUILD_EXECUTABLES)
                 clp-s.cpp
                 ${CLP_S_EXE_SOURCES}
         )
+        target_compile_definitions(
+                clp-s
+                PRIVATE
+                CLP_S_EXCLUDE_LIBCURL=$<BOOL:${CLP_S_EXCLUDE_LIBCURL}>
+        )
         target_compile_features(clp-s PRIVATE cxx_std_20)
         target_include_directories(clp-s PRIVATE ../)
         target_link_libraries(

components/core/src/clp_s/InputConfig.cpp

@@ -14,11 +14,15 @@
 #include <simdjson.h>
 #include <spdlog/spdlog.h>
 
-#include "../clp/aws/AwsAuthenticationSigner.hpp"
+#if !CLP_S_EXCLUDE_LIBCURL
+    #include "../clp/aws/AwsAuthenticationSigner.hpp"
+#endif
 #include "../clp/BufferedReader.hpp"
 #include "../clp/ffi/ir_stream/protocol_constants.hpp"
 #include "../clp/FileReader.hpp"
-#include "../clp/NetworkReader.hpp"
+#if !CLP_S_EXCLUDE_LIBCURL
+    #include "../clp/NetworkReader.hpp"
+#endif
 #include "../clp/ReaderInterface.hpp"
 #include "../clp/spdlog_with_specializations.hpp"
 #include "../clp/streaming_compression/Decompressor.hpp"
@@ -155,6 +159,15 @@ auto try_create_file_reader(std::string_view const file_path)
     }
 }
 
+#if CLP_S_EXCLUDE_LIBCURL
+auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth)
+        -> std::shared_ptr<clp::ReaderInterface> {
+    std::ignore = url;
+    std::ignore = auth;
+    SPDLOG_ERROR("This build of clp-s does not support network reading (libcurl excluded).");
+    return nullptr;
+}
+#else
 auto try_sign_url(std::string& url) -> bool {
     auto const aws_access_key = std::getenv(cAwsAccessKeyIdEnvVar);
     auto const aws_secret_access_key = std::getenv(cAwsSecretAccessKeyEnvVar);
@@ -213,6 +226,7 @@ auto try_create_network_reader(std::string_view const url, NetworkAuthOption con
         return nullptr;
     }
 }
+#endif
 
 auto could_be_zstd(char const* peek_buf, size_t peek_size) -> bool {
     constexpr std::array<char, 4> cZstdMagicNumber = {'\x28', '\xB5', '\x2F', '\xFD'};

components/core/src/clp_s/JsonParser.cpp

@@ -13,7 +13,6 @@
 
 #include <absl/container/flat_hash_map.h>
 #include <boost/uuid/uuid_io.hpp>
-#include <curl/curl.h>
 #include <fmt/format.h>
 #include <simdjson.h>
 #include <spdlog/spdlog.h>
@@ -27,7 +26,6 @@
 #include <clp/ffi/KeyValuePairLogEvent.hpp>
 #include <clp/ffi/SchemaTree.hpp>
 #include <clp/ffi/Value.hpp>
-#include <clp/NetworkReader.hpp>
 #include <clp/ReaderInterface.hpp>
 #include <clp/time_types.hpp>
 #include <clp_s/archive_constants.hpp>
@@ -37,6 +35,7 @@
 #include <clp_s/JsonFileIterator.hpp>
 #include <clp_s/search/ast/ColumnDescriptor.hpp>
 #include <clp_s/search/ast/SearchUtils.hpp>
+#include <clp_s/Utils.hpp>
 
 using clp::ffi::ir_stream::Deserializer;
 using clp::ffi::ir_stream::IRErrorCode;
@@ -672,15 +671,17 @@ bool JsonParser::ingest() {
             case FileType::Zstd:
             case FileType::Unknown:
             default: {
-                std::ignore = check_and_log_curl_error(path, reader);
+                std::ignore = NetworkUtils::check_and_log_curl_error(path.path, reader.get());
                 SPDLOG_ERROR("Could not deduce content type for input {}", path.path);
                 std::ignore = m_archive_writer->close();
                 return false;
             }
         }
 
         close_nested_readers(nested_readers);
-        if (false == ingestion_successful || check_and_log_curl_error(path, reader)) {
+        if (false == ingestion_successful
+            || NetworkUtils::check_and_log_curl_error(path.path, reader.get()))
+        {
             std::ignore = m_archive_writer->close();
             return false;
         }
@@ -1351,27 +1352,4 @@ void JsonParser::split_archive() {
     m_archive_options.id = m_generator();
     m_archive_writer->open(m_archive_options);
 }
-
-bool JsonParser::check_and_log_curl_error(
-        Path const& path,
-        std::shared_ptr<clp::ReaderInterface> reader
-) {
-    if (auto network_reader = std::dynamic_pointer_cast<clp::NetworkReader>(reader);
-        nullptr != network_reader)
-    {
-        if (auto const rc = network_reader->get_curl_ret_code();
-            rc.has_value() && CURLcode::CURLE_OK != rc.value())
-        {
-            auto const curl_error_message = network_reader->get_curl_error_msg();
-            SPDLOG_ERROR(
-                    "Encountered curl error while ingesting {} - Code: {} - Message: {}",
-                    path.path,
-                    static_cast<int64_t>(rc.value()),
-                    curl_error_message.value_or("Unknown error")
-            );
-            return true;
-        }
-    }
-    return false;
-}
 }  // namespace clp_s

components/core/src/clp_s/JsonParser.hpp

@@ -211,16 +211,6 @@ class JsonParser {
      */
     int32_t add_metadata_field(std::string_view const field_name, NodeType type);
 
-    /**
-     * Checks if a reader interface is a clp::NetworkReader that has encountered a CURL error and
-     * logs relevant CURL error information if a CURL error has occurred.
-     * @param path
-     * @param reader
-     * @return true if the provided ReaderInterface has experienced a CURL error and false otherwise
-     */
-    static bool
-    check_and_log_curl_error(Path const& path, std::shared_ptr<clp::ReaderInterface> reader);
-
     std::vector<Path> m_input_paths;
     NetworkAuthOption m_network_auth{};
 

components/core/src/clp_s/Utils.cpp

@@ -11,6 +11,9 @@
 #include <spdlog/spdlog.h>
 #include <string_utils/string_utils.hpp>
 
+#if !CLP_S_EXCLUDE_LIBCURL
+    #include "../clp/NetworkReader.hpp"
+#endif
 #include "archive_constants.hpp"
 
 using std::string;
@@ -149,6 +152,37 @@ bool FileUtils::get_last_non_empty_path_component(std::string_view const path, s
     return false;
 }
 
+#if CLP_S_EXCLUDE_LIBCURL
+auto
+NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader)
+        -> bool {
+    std::ignore = path;
+    std::ignore = reader;
+    return false;
+}
+#else
+auto
+NetworkUtils::check_and_log_curl_error(std::string_view path, clp::ReaderInterface const* reader)
+        -> bool {
+    auto const* network_reader = dynamic_cast<clp::NetworkReader const*>(reader);
+    if (nullptr == network_reader) {
+        return false;
+    }
+    if (auto const curl_error_info = network_reader->get_curl_error_info();
+        curl_error_info.has_value())
+    {
+        SPDLOG_ERROR(
+                "Encountered curl error while reading {} - Code: {} - Message: {}",
+                path,
+                static_cast<int64_t>(curl_error_info->code),
+                curl_error_info->message
+        );
+        return true;
+    }
+    return false;
+}
+#endif
+
 bool UriUtils::get_last_uri_component(std::string_view const uri, std::string& name) {
     auto parsed_result = boost::urls::parse_uri(uri);
     if (false == parsed_result.has_value()) {