introduce clickhouse plugin

extra_plugins/output/clickhouse/CMakeLists.txt

@@ -0,0 +1,87 @@
+cmake_minimum_required(VERSION 3.12)
+project(ipfixcol2-clichouse-output)
+
+# Description of the project
+set(CH_DESCRIPTION
+    "Output plugin for IPFIXcol2 that store flow records to ClickHouse database."
+)
+
+set(CH_VERSION_MAJOR 1)
+set(CH_VERSION_MINOR 0)
+set(CH_VERSION_PATCH 0)
+set(CH_VERSION
+    ${CH_VERSION_MAJOR}.${CH_VERSION_MINOR}.${CH_VERSION_PATCH})
+
+include(CMakeModules/install_dirs.cmake)
+include(CMakeModules/clickhouse-cpp.cmake)
+include(CMakeModules/fmt.cmake)
+include(CheckCCompilerFlag)
+include(CheckCXXCompilerFlag)
+# Include custom FindXXX modules
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
+
+# Find IPFIXcol
+find_package(IPFIXcol2 2.3.0 REQUIRED)
+
+# Set default build type if not specified by user
+if (NOT CMAKE_BUILD_TYPE)
+    set (CMAKE_BUILD_TYPE Debug
+        CACHE STRING "Choose type of build (Release/Debug)." FORCE)
+endif()
+
+option(ENABLE_DOC_MANPAGE    "Enable manual page building"              ON)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS ON)
+
+# Hard coded definitions
+set(CMAKE_C_FLAGS            "${CMAKE_C_FLAGS} -fvisibility=hidden")
+set(CMAKE_C_FLAGS_RELEASE    "-O2 -DNDEBUG")
+set(CMAKE_C_FLAGS_DEBUG      "-g -O0 -Wall -Wextra -pedantic")
+set(CMAKE_CXX_FLAGS          "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+set(CMAKE_CXX_FLAGS_RELEASE  "-O2 -DNDEBUG")
+set(CMAKE_CXX_FLAGS_DEBUG    "-g -O0 -Wall -Wextra -pedantic")
+
+# Header files for source code building
+include_directories(
+    "${IPFIXCOL2_INCLUDE_DIRS}"  # IPFIXcol2 header files
+)
+
+# Create a linkable module
+add_library(clickhouse-output MODULE
+    src/common.cpp
+    src/config.cpp
+    src/datatype.cpp
+    src/main.cpp
+    src/plugin.cpp
+)
+
+target_link_libraries(clickhouse-output PRIVATE clickhouse::client fmt::fmt)
+
+install(
+    TARGETS clickhouse-output
+    LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/"
+)
+
+if (ENABLE_DOC_MANPAGE)
+    find_package(Rst2Man)
+    if (NOT RST2MAN_FOUND)
+        message(FATAL_ERROR "rst2man is not available. Install python-docutils or disable manual page generation (-DENABLE_DOC_MANPAGE=False)")
+    endif()
+
+    # Build a manual page
+    set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-clickhouse-output.7.rst")
+    set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-clickhouse-output.7")
+
+    add_custom_command(TARGET clickhouse-output PRE_BUILD
+        COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE}
+        DEPENDS ${SRC_FILE}
+        VERBATIM
+    )
+
+    install(
+        FILES "${DST_FILE}"
+        DESTINATION "${INSTALL_DIR_MAN}/man7"
+    )
+endif()

extra_plugins/output/clickhouse/CMakeModules/FindIPFIXcol2.cmake

@@ -0,0 +1,39 @@
+#  IPFIXCOL2_FOUND        - System has IPFIXcol
+#  IPFIXCOL2_INCLUDE_DIRS - The IPFIXcol include directories
+#  IPFIXCOL2_DEFINITIONS  - Compiler switches required for using IPFIXcol
+
+# use pkg-config to get the directories and then use these values
+# in the find_path() and find_library() calls
+find_package(PkgConfig)
+pkg_check_modules(PC_IPFIXCOL QUIET ipfixcol2)
+set(IPFIXCOL2_DEFINITIONS ${PC_IPFIXCOL_CFLAGS_OTHER})
+
+find_path(
+	IPFIXCOL2_INCLUDE_DIR ipfixcol2.h
+	HINTS ${PC_IPFIXCOL_INCLUDEDIR} ${PC_IPFIXCOL_INCLUDE_DIRS}
+	PATH_SUFFIXES include
+)
+
+if (PC_IPFIXCOL_VERSION)
+    # Version extracted from pkg-config
+    set(IPFIXCOL_VERSION_STRING ${PC_IPFIXCOL_VERSION})
+elseif(IPFIXCOL2_INCLUDE_DIR AND EXISTS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h")
+    # Try to extract library version from a header file
+    file(STRINGS "${IPFIXCOL2_INCLUDE_DIR}/ipfixcol2/api.h" ipfixcol_version_str
+         REGEX "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\".*\"")
+
+    string(REGEX REPLACE "^#define[\t ]+IPX_API_VERSION_STR[\t ]+\"([^\"]*)\".*" "\\1"
+		IPFIXCOL_VERSION_STRING "${ipfixcol_version_str}")
+    unset(ipfixcol_version_str)
+endif()
+
+# handle the QUIETLY and REQUIRED arguments and set IPFIXCOL2_FOUND to TRUE
+# if all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(IPFIXcol2
+	REQUIRED_VARS IPFIXCOL2_INCLUDE_DIR
+	VERSION_VAR IPFIXCOL_VERSION_STRING
+)
+
+set(IPFIXCOL2_INCLUDE_DIRS ${IPFIXCOL2_INCLUDE_DIR})
+mark_as_advanced(IPFIXCOL2_INCLUDE_DIR)

extra_plugins/output/clickhouse/CMakeModules/FindRst2Man.cmake

@@ -0,0 +1,30 @@
+#  RST2MAN_FOUND - true if the program was found
+#  RST2MAN_VERSION - version of rst2man
+#  RST2MAN_EXECUTABLE - path to the rst2man program
+
+find_program(RST2MAN_EXECUTABLE
+	NAMES rst2man rst2man.py rst2man-3 rst2man-3.py
+	DOC "The Python Docutils generator of Unix Manpages from reStructuredText"
+)
+
+if (RST2MAN_EXECUTABLE)
+	# Get the version string
+	execute_process(
+		COMMAND ${RST2MAN_EXECUTABLE} --version
+		OUTPUT_VARIABLE rst2man_version_str
+	)
+	# Expected format: rst2man (Docutils 0.13.1 [release], Python 2.7.15, on linux2)
+	string(REGEX REPLACE "^rst2man[\t ]+\\(Docutils[\t ]+([^\t ]*).*" "\\1"
+		RST2MAN_VERSION "${rst2man_version_str}")
+	unset(rst2man_version_str)
+endif()
+
+# handle the QUIETLY and REQUIRED arguments and set RST2MAN_FOUND to TRUE
+# if all listed variables are set
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Rst2Man
+	REQUIRED_VARS RST2MAN_EXECUTABLE
+	VERSION_VAR RST2MAN_VERSION
+)
+
+mark_as_advanced(RST2MAN_EXECUTABLE RST2MAN_VERSION)

extra_plugins/output/clickhouse/CMakeModules/clickhouse-cpp.cmake

@@ -0,0 +1,19 @@
+# clickhouse-cpp library (C++ client for ClickHouse)
+#
+# The project consists of a library that can be independently
+# added as dependency:
+#  - clickhouse::client
+
+include(FetchContent)
+set(FETCHCONTENT_QUIET OFF)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+FetchContent_Declare(
+        clickhouse
+        GIT_REPOSITORY "https://github.com/ClickHouse/clickhouse-cpp.git"
+        GIT_TAG "v2.5.1"
+        GIT_SHALLOW ON
+)
+
+FetchContent_MakeAvailable(clickhouse)
+add_library(clickhouse::client ALIAS clickhouse-cpp-lib)

extra_plugins/output/clickhouse/CMakeModules/fmt.cmake

@@ -0,0 +1,18 @@
+# fmt library
+#
+# The project consists of a library that can be independently
+# added as dependency:
+#  - fmt::fmt
+
+include(FetchContent)
+set(FETCHCONTENT_QUIET OFF)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+FetchContent_Declare(
+        fmt
+        GIT_REPOSITORY "https://github.com/fmtlib/fmt"
+        GIT_TAG "11.0.2"
+        GIT_SHALLOW ON
+)
+
+FetchContent_MakeAvailable(fmt)

extra_plugins/output/clickhouse/CMakeModules/install_dirs.cmake

@@ -0,0 +1,43 @@
+# The purpose of this file is to automatically determine install directories
+#
+# If no directories are defined, use GNU install directories by default.
+# However, in case of RPM build, install directories are typically passed
+# to CMake as definitions that overwrites the default paths.
+#
+
+include(GNUInstallDirs)
+
+# Binary directories
+set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR})
+
+# Library directories
+if (DEFINED LIB_INSTALL_DIR)
+    set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR})
+endif()
+
+# Include directories
+if (DEFINED INCLUDE_INSTALL_DIR)
+    set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR})
+endif()
+
+# System configuration
+if (DEFINED SYSCONF_INSTALL_DIR)
+    set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR})
+endif()
+
+# Share files (architecture independend data)
+if (DEFINED SHARE_INSTALL_PREFIX)
+    set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX})
+else()
+    set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR})
+endif()
+
+set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/")
+set(INSTALL_DIR_MAN  "${INSTALL_DIR_SHARE}/man/")
+set(INSTALL_DIR_DOC  "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/")

extra_plugins/output/clickhouse/doc/ipfixcol2-clickhouse-output.7.rst

@@ -0,0 +1,24 @@
+=============================
+ ipfixcol2-clickhouse-output
+=============================
+
+--------------------------
+ClickHouse (output plugin)
+--------------------------
+
+:Author: Michal Sedlak (sedlakm@cesnet.cz)
+:Date:   2024-11-04
+:Copyright: Copyright © 2024 CESNET, z.s.p.o.
+:Version: 1.0
+:Manual section: 7
+:Manual group: IPFIXcol collector
+
+Description
+-----------
+
+.. .. include:: ../README.rst
+..    :start-line: 3
+..    :end-before: How to build
+..
+.. .. include:: ../README.rst
+..    :start-after: make install

extra_plugins/output/clickhouse/src/common.cpp

@@ -0,0 +1,63 @@
+/**
+ * @file
+ * @author Michal Sedlak <sedlakm@cesnet.cz>
+ * @brief
+ * @date 2024
+ *
+ * Copyright(c) 2024 CESNET z.s.p.o.
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "common.h"
+
+#include <iostream>
+
+void print_block(const clickhouse::Block& block) {
+    std::cout << "================================================================================\n";
+    for (size_t i = 0; i < block.GetColumnCount(); i++) {
+        std::cout << block.GetColumnName(i) << ":" << block[i]->GetType().GetName() << (i < block.GetColumnCount() - 1 ? '\t' : '\n');
+    }
+    std::cout << "--------------------------------------------------------------------------------\n";
+
+    for (size_t i = 0; i < block.GetRowCount(); i++) {
+        for (size_t j = 0; j < block.GetColumnCount(); j++) {
+            switch (block[j]->GetType().GetCode()) {
+            case clickhouse::Type::Int8:
+                std::cout << +block[j]->As<clickhouse::ColumnInt8>()->At(i);
+                break;
+            case clickhouse::Type::Int16:
+                std::cout << block[j]->As<clickhouse::ColumnInt16>()->At(i);
+                break;
+            case clickhouse::Type::Int32:
+                std::cout << block[j]->As<clickhouse::ColumnInt32>()->At(i);
+                break;
+            case clickhouse::Type::Int64:
+                std::cout << block[j]->As<clickhouse::ColumnInt64>()->At(i);
+                break;
+            case clickhouse::Type::Int128:
+                std::cout << block[j]->As<clickhouse::ColumnInt128>()->At(i);
+                break;
+            case clickhouse::Type::UInt8:
+                std::cout << +block[j]->As<clickhouse::ColumnUInt8>()->At(i);
+                break;
+            case clickhouse::Type::UInt16:
+                std::cout << block[j]->As<clickhouse::ColumnUInt16>()->At(i);
+                break;
+            case clickhouse::Type::UInt32:
+                std::cout << block[j]->As<clickhouse::ColumnUInt32>()->At(i);
+                break;
+            case clickhouse::Type::UInt64:
+                std::cout << block[j]->As<clickhouse::ColumnUInt64>()->At(i);
+                break;
+            case clickhouse::Type::String:
+                std::cout << block[j]->As<clickhouse::ColumnString>()->At(i);
+                break;
+            default:
+                std::cout << "-";
+                break;
+            }
+            std::cout << (j < block.GetColumnCount() - 1 ? '\t' : '\n');
+        }
+    }
+    std::cout << "================================================================================\n\n";
+};