diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index adb2a7f2330e..93708415294a 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -288,6 +288,18 @@ jobs: test -e /usr/local/lib/suricata/python/suricata/update/configs/modify.conf test -e /usr/local/lib/suricata/python/suricata/update/configs/threshold.in test -e /usr/local/lib/suricata/python/suricata/update/configs/update.yaml + - name: Build C json filetype plugin + working-directory: examples/plugins/c-json-filetype + run: make + - name: Check C json filetype plugin + run: test -e examples/plugins/c-json-filetype/.libs/json-filetype.so.0.0.0 + - name: Installing headers and library + run: | + make install-headers + make install-library + - name: Test plugin build with Makefile.example + working-directory: examples/plugins/c-json-filetype + run: PATH=/usr/local/bin:$PATH make -f Makefile.example almalinux-9-templates: name: AlmaLinux 9 Test Templates diff --git a/Makefile.am b/Makefile.am index 67963ed32fcf..b7a221f49299 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \ scripts/generate-images.sh SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib etc python ebpf \ $(SURICATA_UPDATE_DIR) +DIST_SUBDIRS = examples/plugins/c-json-filetype $(SUBDIRS) CLEANFILES = stamp-h[0-9]* diff --git a/configure.ac b/configure.ac index 1908c76ab76a..1eb054d98c1e 100644 --- a/configure.ac +++ b/configure.ac @@ -2613,6 +2613,8 @@ AC_CONFIG_FILES(suricata.yaml etc/Makefile etc/suricata.logrotate etc/suricata.s AC_CONFIG_FILES(python/Makefile python/suricata/config/defaults.py) AC_CONFIG_FILES(ebpf/Makefile) AC_CONFIG_FILES(libsuricata-config) +AC_CONFIG_FILES(examples/plugins/c-json-filetype/Makefile) + AC_OUTPUT SURICATA_BUILD_CONF="Suricata Configuration: diff --git a/examples/plugins/README.md b/examples/plugins/README.md new file mode 100644 index 000000000000..8e47b6e7ffc2 --- /dev/null +++ b/examples/plugins/README.md @@ -0,0 +1,6 @@ +# Example Plugins + +## c-json-filetype + +An example plugin of an EVE/JSON filetype plugin. This type of plugin +is useful if you want to send EVE output to custom destinations. diff --git a/examples/plugins/c-json-filetype/.gitignore b/examples/plugins/c-json-filetype/.gitignore new file mode 100644 index 000000000000..f5bb3af73f70 --- /dev/null +++ b/examples/plugins/c-json-filetype/.gitignore @@ -0,0 +1,2 @@ +*.so +*.la diff --git a/examples/plugins/c-json-filetype/Makefile.am b/examples/plugins/c-json-filetype/Makefile.am new file mode 100644 index 000000000000..d5e912b8a469 --- /dev/null +++ b/examples/plugins/c-json-filetype/Makefile.am @@ -0,0 +1,17 @@ +plugindir = ${libdir}/suricata/plugins + +if BUILD_SHARED_LIBRARY +plugin_LTLIBRARIES = json-filetype.la +json_filetype_la_LDFLAGS = -module -shared +json_filetype_la_SOURCES = filetype.c + +json_filetype_la_CPPFLAGS = -I$(abs_top_srcdir)/rust/gen -I$(abs_top_srcdir)/rust/dist + +else + +all-local: + @echo + @echo "Shared library support must be enabled to build plugins." + @echo + +endif diff --git a/examples/plugins/c-json-filetype/Makefile.example b/examples/plugins/c-json-filetype/Makefile.example new file mode 100644 index 000000000000..6d514aab1445 --- /dev/null +++ b/examples/plugins/c-json-filetype/Makefile.example @@ -0,0 +1,18 @@ +SRCS := filetype.c + +LIBSURICATA_CONFIG ?= libsuricata-config + +CPPFLAGS += `$(LIBSURICATA_CONFIG) --cflags` +CPPFLAGS += -DSURICATA_PLUGIN -I. +CPPFLAGS += "-D__SCFILENAME__=\"$(*F)\"" + +OBJS := $(SRCS:.c=.o) + +filetype.so: $(OBJS) + $(CC) -fPIC -shared -o $@ $(OBJS) + +%.o: %.c + $(CC) -fPIC $(CPPFLAGS) -c -o $@ $< + +clean: + rm -f *.o *.so *~ diff --git a/examples/plugins/c-json-filetype/README.md b/examples/plugins/c-json-filetype/README.md new file mode 100644 index 000000000000..2f7978977dbd --- /dev/null +++ b/examples/plugins/c-json-filetype/README.md @@ -0,0 +1,123 @@ +# Example EVE Filetype Plugin + +## Building + +If in the Suricata source directory, this plugin can be built by +running `make` and installed with `make install`. + +Note that Suricata must have been built without `--disable-shared`. + +## Building Standalone + +The file `Makefile.example` is an example of how you might build a +plugin that is distributed separately from the Suricata source code. + +It has the following dependencies: + +- Suricata is installed +- The Suricata library is installed: `make install-library` +- The Suricata development headers are installed: `make install-headers` +- The program `libsuricata-config` is in your path (installed with + `make install-library`) + +The run: `make -f Makefile.example` + +Before building this plugin you will need to build and install Suricata from the +git master branch and install the development tools and headers: + +- `make install-library` +- `make install-headers` + +then make sure the newly installed tool `libsuricata-config` can be +found in your path, for example: +``` +libsuricata-config --cflags +``` + +Then a simple `make` should build this plugin. + +Or if the Suricata installation is not in the path, a command like the following +can be used: + +``` +PATH=/opt/suricata/bin:$PATH make +``` + +## Usage + +To run the plugin, first add the path to the plugin you just compiled to +your `suricata.yaml`, for example: +``` +plugins: + - /usr/lib/suricata/plugins/json-filetype.so +``` + +Then add an output for the plugin: +``` +outputs: + - eve-log: + enabled: yes + filetype: json-filetype-plugin + threaded: true + types: + - dns + - tls + - http +``` + +In the example above we use the name specified in the plugin as the `filetype` +and specify that all `dns`, `tls` and `http` log entries should be sent to the +plugin. + +## Details + +This plugin demonstrates a Suricata JSON/EVE output plugin +(file-type). The idea of a Suricata EVE output plugin is to provide a +file like interface for the handling of rendered JSON logs. This is +useful for custom destinations not builtin to Suricata or if the +formatted JSON requires some post-processing. + +Note: EVE output plugins are not that useful just for reformatting the +JSON output as the plugin does need to handle writing to a file once +the file type has been delegated to the plugin. + +### Registering a Plugin + +All Suricata plugins make themselves known to Suricata by using a +function named `SCPluginRegister` which is called after Suricata loads +the plugin shared object file. This function must return a `SCPlugin` +struct which contains basic information about the plugin. For +example: + +```c +const SCPlugin PluginRegistration = { + .name = "eve-filetype", + .author = "Jason Ish", + .license = "GPLv2", + .Init = TemplateInit, +}; + +const SCPlugin *SCPluginRegister() { + return &PluginRegistration; +} +``` + +### Initializing a Plugin + +After the plugin has been registered, the `Init` callback will be called. This +is where the plugin will set itself up as a specific type of plugin such as an +EVE output, or a capture method. + +This plugins registers itself as an EVE file type using the +`SCRegisterEveFileType` struct. To register as an EVE file type the +following must be provided: + +* name: This is the name of the output which will be used in the eve filetype + field in `suricata.yaml` to enable this output. +* Init: The callback called when the output is "opened". +* Deinit: The callback called the output is "closed". +* ThreadInit: Callback called to initialize per thread data (if threaded). +* ThreadDeinit: Callback called to deinitialize per thread data (if threaded). +* Write: The callback called when an EVE record is to be "written". + +Please see the code in `filetype.c` for more details about this functions. diff --git a/examples/plugins/c-json-filetype/filetype.c b/examples/plugins/c-json-filetype/filetype.c new file mode 100644 index 000000000000..9c81d7f03267 --- /dev/null +++ b/examples/plugins/c-json-filetype/filetype.c @@ -0,0 +1,243 @@ +/* Copyright (C) 2020-2023 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "suricata-plugin.h" +#include "util-mem.h" +#include "util-debug.h" + +#define FILETYPE_NAME "json-filetype-plugin" + +static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data); +static int FiletypeThreadDeinit(void *ctx, void *thread_data); + +/** + * Per thread context data for each logging thread. + */ +typedef struct ThreadData_ { + /** The thread ID, for demonstration purposes only. */ + int thread_id; + + /** The number of records logged on this thread. */ + uint64_t count; +} ThreadData; + +/** + * A context object for each eve logger using this output. + */ +typedef struct Context_ { + /** Verbose, or print to stdout. */ + int verbose; + + /** A thread context to use when not running in threaded mode. */ + ThreadData *thread; +} Context; + +/** + * This function is called to initialize the output, it can be somewhat thought + * of like opening a file. + * + * \param conf The EVE configuration node using this output. + * + * \param threaded If true the EVE subsystem is running in threaded mode. + * + * \param data A pointer where context data can be stored relevant to this + * output. + * + * Eve output plugins need to be thread aware as the threading happens at lower + * level than the EVE output, so a flag is provided here to notify the plugin if + * threading is enabled or not. + * + * If the plugin does not work with threads disabled, or enabled, this function + * should return -1. + * + * Note for upgrading a plugin from 6.0 to 7.0: The ConfNode in 7.0 is the + * configuration for the eve instance, not just a node named after the plugin. + * This allows the plugin to get more context about what it is logging. + */ +static int FiletypeInit(ConfNode *conf, bool threaded, void **data) +{ + SCLogNotice("Initializing template eve output plugin: threaded=%d", threaded); + Context *context = SCCalloc(1, sizeof(Context)); + if (context == NULL) { + return -1; + } + + /* Verbose by default. */ + int verbose = 1; + + /* An example of how you can access configuration data from a + * plugin. */ + if (conf && (conf = ConfNodeLookupChild(conf, "eve-template")) != NULL) { + if (!ConfGetChildValueBool(conf, "verbose", &verbose)) { + verbose = 1; + } else { + SCLogNotice("Read verbose configuration value of %d", verbose); + } + } + context->verbose = verbose; + + if (!threaded) { + /* We're not running in threaded mode so allocate a thread context here + * to avoid duplication of context data such as file pointers, database + * connections, etc. */ + if (FiletypeThreadInit(context, 0, (void **)&context->thread) != 0) { + SCFree(context); + return -1; + } + } + *data = context; + return 0; +} + +/** + * This function is called when the output is closed. + * + * This will be called after ThreadDeinit is called for each thread. + * + * \param data The data allocated in FiletypeInit. It should be cleaned up and + * deallocated here. + */ +static void FiletypeDeinit(void *data) +{ + printf("TemplateClose\n"); + Context *ctx = data; + if (ctx != NULL) { + if (ctx->thread) { + FiletypeThreadDeinit(ctx, (void *)ctx->thread); + } + SCFree(ctx); + } +} + +/** + * Initialize per thread context. + * + * \param ctx The context created in TemplateInitOutput. + * + * \param thread_id An identifier for this thread. + * + * \param thread_data Pointer where thread specific context can be stored. + * + * When the EVE output is running in threaded mode this will be called once for + * each output thread with a unique thread_id. For regular file logging in + * threaded mode Suricata uses the thread_id to construct the files in the form + * of "eve..json". This plugin may want to do similar, or open + * multiple connections to whatever the final logging location might be. + * + * In the case of non-threaded EVE logging this function is NOT called by + * Suricata, but instead this plugin chooses to use this method to create a + * default (single) thread context. + */ +static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data) +{ + ThreadData *tdata = SCCalloc(1, sizeof(ThreadData)); + if (tdata == NULL) { + SCLogError("Failed to allocate thread data"); + return -1; + } + tdata->thread_id = thread_id; + *thread_data = tdata; + SCLogNotice( + "Initialized thread %03d (pthread_id=%" PRIuMAX ")", tdata->thread_id, pthread_self()); + return 0; +} + +/** + * Deinitialize a thread. + * + * This is where any cleanup per thread should be done including free'ing of the + * thread_data if needed. + */ +static int FiletypeThreadDeinit(void *ctx, void *thread_data) +{ + if (thread_data == NULL) { + // Nothing to do. + return 0; + } + + ThreadData *tdata = thread_data; + SCLogNotice( + "Deinitializing thread %d: records written: %" PRIu64, tdata->thread_id, tdata->count); + SCFree(tdata); + return 0; +} + +/** + * This method is called with formatted Eve JSON data. + * + * \param buffer Formatted JSON buffer \param buffer_len Length of formatted + * JSON buffer \param data Data set in Init callback \param thread_data Data set + * in ThreadInit callbacl + * + * Do not block in this thread, it will cause packet loss. Instead of outputting + * to any resource that may block it might be best to enqueue the buffers for + * further processing which will require copying of the provided buffer. + */ +static int FiletypeWrite(const char *buffer, int buffer_len, void *data, void *thread_data) +{ + Context *ctx = data; + ThreadData *thread = thread_data; + + /* The thread_data could be null which is valid, or it could be that we are + * in single threaded mode. */ + if (thread == NULL) { + thread = ctx->thread; + } + + thread->count++; + + if (ctx->verbose) { + SCLogNotice("Received write with thread_data %p: %s", thread_data, buffer); + } + return 0; +} + +/** + * Called by Suricata to initialize the module. This module registers + * new file type to the JSON logger. + */ +void PluginInit(void) +{ + SCEveFileType *my_output = SCCalloc(1, sizeof(SCEveFileType)); + my_output->name = FILETYPE_NAME; + my_output->Init = FiletypeInit; + my_output->Deinit = FiletypeDeinit; + my_output->ThreadInit = FiletypeThreadInit; + my_output->ThreadDeinit = FiletypeThreadDeinit; + my_output->Write = FiletypeWrite; + if (!SCRegisterEveFileType(my_output)) { + FatalError("Failed to register filetype plugin: %s", FILETYPE_NAME); + } +} + +const SCPlugin PluginRegistration = { + .name = FILETYPE_NAME, + .author = "FirstName LastName ", + .license = "GPL-2.0-only", + .Init = PluginInit, +}; + +/** + * The function called by Suricata after loading this plugin. + * + * A pointer to a populated SCPlugin struct must be returned. + */ +const SCPlugin *SCPluginRegister() +{ + return &PluginRegistration; +}