From e60e032605f7d5bbf38280041e4b8471309e8b10 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 8 May 2024 21:32:58 -0600 Subject: [PATCH] Initial commit. zkg 2.14.0 created this package from template "package-template" using version v3.1.0, with features "github-ci", "license" and "spicy-packet-analyzer". --- .github/workflows/zeek-matrix.yml | 24 ++++++ .github/workflows/zeek-nightly.yml | 19 +++++ CMakeLists.txt | 18 ++++ COPYING | 18 ++++ README | 4 + analyzer/CMakeLists.txt | 6 ++ analyzer/protobuf.evt | 12 +++ analyzer/protobuf.spicy | 19 +++++ analyzer/zeek_protobuf.spicy | 8 ++ cmake/FindSpicyPlugin.cmake | 78 ++++++++++++++++++ scripts/__load__.zeek | 1 + scripts/main.zeek | 38 +++++++++ testing/.gitignore | 2 + .../Baseline/tests.run-pcap/conn.log.filtered | 2 + testing/Baseline/tests.run-pcap/output | 2 + .../Baseline/tests.trace/conn.log.filtered | 4 + testing/Baseline/tests.trace/output | 2 + testing/Files/random.seed | 21 +++++ testing/Makefile | 3 + testing/Scripts/README | 7 ++ testing/Scripts/diff-remove-timestamps | 13 +++ testing/Scripts/get-zeek-env | 40 +++++++++ testing/Traces/raw-layer.pcap | Bin 0 -> 161 bytes testing/btest.cfg | 27 ++++++ testing/tests/availability.zeek | 3 + testing/tests/trace.zeek | 19 +++++ zkg.meta | 23 ++++++ 27 files changed, 413 insertions(+) create mode 100644 .github/workflows/zeek-matrix.yml create mode 100644 .github/workflows/zeek-nightly.yml create mode 100644 CMakeLists.txt create mode 100644 COPYING create mode 100644 README create mode 100644 analyzer/CMakeLists.txt create mode 100644 analyzer/protobuf.evt create mode 100644 analyzer/protobuf.spicy create mode 100644 analyzer/zeek_protobuf.spicy create mode 100644 cmake/FindSpicyPlugin.cmake create mode 100644 scripts/__load__.zeek create mode 100644 scripts/main.zeek create mode 100644 testing/.gitignore create mode 100644 testing/Baseline/tests.run-pcap/conn.log.filtered create mode 100644 testing/Baseline/tests.run-pcap/output create mode 100644 testing/Baseline/tests.trace/conn.log.filtered create mode 100644 testing/Baseline/tests.trace/output create mode 100644 testing/Files/random.seed create mode 100644 testing/Makefile create mode 100644 testing/Scripts/README create mode 100755 testing/Scripts/diff-remove-timestamps create mode 100755 testing/Scripts/get-zeek-env create mode 100644 testing/Traces/raw-layer.pcap create mode 100644 testing/btest.cfg create mode 100644 testing/tests/availability.zeek create mode 100644 testing/tests/trace.zeek create mode 100644 zkg.meta diff --git a/.github/workflows/zeek-matrix.yml b/.github/workflows/zeek-matrix.yml new file mode 100644 index 0000000..406c01f --- /dev/null +++ b/.github/workflows/zeek-matrix.yml @@ -0,0 +1,24 @@ +name: Zeek matrix tests + +on: + push: + pull_request: + +jobs: + test: + name: test-${{ matrix.zeekver }} + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + zeekver: [zeek, zeek-lts, zeek-nightly] + steps: + - uses: actions/checkout@v3 + - uses: zeek/action-zkg-install@v2 + with: + zeek_version: ${{ matrix.zeekver }} + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: zkg-logs-${{ matrix.zeekver }} + path: ${{ github.workspace }}/.action-zkg-install/artifacts diff --git a/.github/workflows/zeek-nightly.yml b/.github/workflows/zeek-nightly.yml new file mode 100644 index 0000000..c41178e --- /dev/null +++ b/.github/workflows/zeek-nightly.yml @@ -0,0 +1,19 @@ +name: Zeek nightly build + +on: + schedule: + - cron: 0 0 * * * + +jobs: + test-nightly: + runs-on: ubuntu-20.04 + steps: + - uses: zeek/action-zkg-install@v2 + with: + pkg: ${{ github.server_url }}/${{ github.repository }} + zeek_version: zeek-nightly + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: zkg-logs + path: ${{ github.workspace }}/.action-zkg-install/artifacts diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..88895c2 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) + +project(protobuf LANGUAGES C) + +list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +find_package(SpicyPlugin REQUIRED) + +# Set minimum versions that this plugin needs. Make sure to use "x.y.z" format. +# spicy_require_version("1.2.0") +# spicy_plugin_require_version("0.99.0") +# zeek_require_version("5.0.0") + +if (NOT CMAKE_BUILD_TYPE) + # Default to release build. + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") +endif () + +add_subdirectory(analyzer) diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..340c1c3 --- /dev/null +++ b/COPYING @@ -0,0 +1,18 @@ +Copyright 2024 Tanner Kvarfordt tanner.kvarfordt@gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README b/README new file mode 100644 index 0000000..a3f47da --- /dev/null +++ b/README @@ -0,0 +1,4 @@ +spicy-protobuf +================================= + +TODO: tell people what your package does! diff --git a/analyzer/CMakeLists.txt b/analyzer/CMakeLists.txt new file mode 100644 index 0000000..292aed7 --- /dev/null +++ b/analyzer/CMakeLists.txt @@ -0,0 +1,6 @@ +spicy_add_analyzer( + NAME protobuf + PACKAGE_NAME spicy-protobuf + SOURCES protobuf.spicy protobuf.evt zeek_protobuf.spicy + SCRIPTS __load__.zeek main.zeek dpd.sig +) diff --git a/analyzer/protobuf.evt b/analyzer/protobuf.evt new file mode 100644 index 0000000..e06367d --- /dev/null +++ b/analyzer/protobuf.evt @@ -0,0 +1,12 @@ +import protobuf; +import Zeek_protobuf; + +packet analyzer protobuf: + parse with protobuf::Packet; + +# TODO: Connect Spicy-side events with Zeek-side events. The example just +# defines a simple example event that forwards the raw data (which in practice you +# don't want to do!). In fact, you should consider just deleting this event if +# you don't need it: For most packet analyzers, it's best to not do any +# script-level work because the overhead could quickly become overwhelming. +on protobuf::Packet -> event protobuf::packet($packet, self.payload); diff --git a/analyzer/protobuf.spicy b/analyzer/protobuf.spicy new file mode 100644 index 0000000..83e9fdf --- /dev/null +++ b/analyzer/protobuf.spicy @@ -0,0 +1,19 @@ +# TODO: Define your analyzer here. + +module protobuf; + +import zeek; + +# TODO: Our example here models a simple example packet format of static size: +# 19 payload bytes, followed by the protocol number for the next layer, for +# which the data then follows subsequently. (This is just what our test trace +# happens to contain). Adapt as suitable. +public type Packet = unit { + payload: bytes &size=19; + protocol: uint16; # next-layer protocol, values need to be mapped to analyzers in Zeek scriptland + + on %done { + # Feed into Zeek's next-layer packet analysis. + zeek::forward_packet(self.protocol); + } +}; diff --git a/analyzer/zeek_protobuf.spicy b/analyzer/zeek_protobuf.spicy new file mode 100644 index 0000000..be41a18 --- /dev/null +++ b/analyzer/zeek_protobuf.spicy @@ -0,0 +1,8 @@ +# Zeek-specific Spicy logic. + +module Zeek_protobuf; + +import protobuf; +import zeek; + +# TODO: Add anything you need here. diff --git a/cmake/FindSpicyPlugin.cmake b/cmake/FindSpicyPlugin.cmake new file mode 100644 index 0000000..67cbd47 --- /dev/null +++ b/cmake/FindSpicyPlugin.cmake @@ -0,0 +1,78 @@ +# Find the Spicy plugin to get access to the infrastructure it provides. +# +# While most of the actual CMake logic for building analyzers comes with the Spicy +# plugin for Zeek, this code bootstraps us by asking "spicyz" for the plugin's +# location. Either make sure that "spicyz" is in PATH, set the environment +# variable SPICYZ to point to its location, or set variable ZEEK_SPICY_ROOT +# in either CMake or environment to point to its installation or build +# directory. +# +# This exports: +# +# SPICY_PLUGIN_FOUND True if plugin and all dependencies were found +# SPICYZ Path to spicyz +# SPICY_PLUGIN_VERSION Version string of plugin +# SPICY_PLUGIN_VERSION_NUMBER Numerical version number of plugin + +# Runs `spicyz` with the flags given as second argument and stores the output in the variable named +# by the first argument. +function (run_spicycz output) + execute_process(COMMAND "${SPICYZ}" ${ARGN} OUTPUT_VARIABLE output_ + OUTPUT_STRIP_TRAILING_WHITESPACE) + + string(STRIP "${output_}" output_) + set(${output} "${output_}" PARENT_SCOPE) +endfunction () + +# Checks that the Spicy plugin version it at least the given version. +function (spicy_plugin_require_version version) + string(REGEX MATCH "([0-9]*)\.([0-9]*)\.([0-9]*).*" _ ${version}) + math(EXPR version_number "${CMAKE_MATCH_1} * 10000 + ${CMAKE_MATCH_2} * 100 + ${CMAKE_MATCH_3}") + + if ("${SPICY_PLUGIN_VERSION_NUMBER}" LESS "${version_number}") + message(FATAL_ERROR "Package requires at least Spicy plugin version ${version}, " + "have ${SPICY_PLUGIN_VERSION}") + endif () +endfunction () + +### +### Main +### + +if (NOT SPICYZ) + set(SPICYZ "$ENV{SPICYZ}") +endif () + +if (NOT SPICYZ) + # Support an in-tree Spicy build. + find_program( + spicyz spicyz + HINTS ${ZEEK_SPICY_ROOT}/bin ${ZEEK_SPICY_ROOT}/build/bin $ENV{ZEEK_SPICY_ROOT}/bin + $ENV{ZEEK_SPICY_ROOT}/build/bin ${PROJECT_SOURCE_DIR}/../../build/bin) + set(SPICYZ "${spicyz}") +endif () + +message(STATUS "spicyz: ${SPICYZ}") + +if (SPICYZ) + set(SPICYZ "${SPICYZ}" CACHE PATH "" FORCE) # make sure it's in the cache + + run_spicycz(SPICY_PLUGIN_VERSION "--version") + run_spicycz(SPICY_PLUGIN_VERSION_NUMBER "--version-number") + message(STATUS "Zeek plugin version: ${SPICY_PLUGIN_VERSION}") + + run_spicycz(spicy_plugin_path "--print-plugin-path") + set(spicy_plugin_cmake_path "${spicy_plugin_path}/cmake") + message(STATUS "Zeek plugin CMake path: ${spicy_plugin_cmake_path}") + + list(PREPEND CMAKE_MODULE_PATH "${spicy_plugin_cmake_path}") + find_package(Zeek REQUIRED) + find_package(Spicy REQUIRED) + zeek_print_summary() + spicy_print_summary() + + include(ZeekSpicyAnalyzerSupport) +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(SpicyPlugin DEFAULT_MSG SPICYZ ZEEK_FOUND) diff --git a/scripts/__load__.zeek b/scripts/__load__.zeek new file mode 100644 index 0000000..a10fe85 --- /dev/null +++ b/scripts/__load__.zeek @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/main.zeek b/scripts/main.zeek new file mode 100644 index 0000000..cdfd15d --- /dev/null +++ b/scripts/main.zeek @@ -0,0 +1,38 @@ +@load base/misc/version + +module protobuf; + +event zeek_init() &priority=20 + { + # TODO: Our example here models a custom protocol sitting between + # Ethernet and IP. The following sets that up, using a custom ether + # type 0x88b5. Adapt as suitable, some suggestions in comments. + local analyzer = PacketAnalyzer::ANALYZER_PROTOBUF; + + # Activate our analyzer on top of Ethernet. + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x88b5, analyzer); + + # Activate IP on top of our analyzer. 0x4950 is our own protocol's + # magic number indicating that IP comes next. + PacketAnalyzer::register_packet_analyzer(analyzer, 0x4950, PacketAnalyzer::ANALYZER_IP); + + # Alternative: Use this if your analyzer parses a link layer protocol directly. + # const DLT_protobuf : count = 12345; + # PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ROOT, DLT_protobuf, analyzer); + + # Alternative: Use this if your analyzer parses a protocol running on top of + # IPv4, using the specified IP protocol number. + # PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 0xcafe, analyzer); + + # Alternative: Use this if you want your analyzer to run on top of UDP, activated on the specified well-known port. + # const ports: set[port] = { 6789/udp } &redef; + # PacketAnalyzer::register_for_ports(PacketAnalyzer::ANALYZER_UDP, analyzer, ports); + } + +# Example event defined in protobuf.evt. +event protobuf::packet(p: raw_pkt_hdr, payload: string) + { + # TODO: Consider just deleting this event handler if you don't need it. + # For most packet analyzers, it's best to not do any script-level work + # because the overhead could quickly become overwhelming. + } diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 0000000..fc422ef --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,2 @@ +.btest.failed.dat +.tmp diff --git a/testing/Baseline/tests.run-pcap/conn.log.filtered b/testing/Baseline/tests.run-pcap/conn.log.filtered new file mode 100644 index 0000000..1e6917b --- /dev/null +++ b/testing/Baseline/tests.run-pcap/conn.log.filtered @@ -0,0 +1,2 @@ +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.9.63 63526 54.175.222.246 80 tcp http 0.755677 207 489 SF 0 ShADTadFf 7 790 4 705 - diff --git a/testing/Baseline/tests.run-pcap/output b/testing/Baseline/tests.run-pcap/output new file mode 100644 index 0000000..781c9fc --- /dev/null +++ b/testing/Baseline/tests.run-pcap/output @@ -0,0 +1,2 @@ +Hello world! +Goodbye world! diff --git a/testing/Baseline/tests.trace/conn.log.filtered b/testing/Baseline/tests.trace/conn.log.filtered new file mode 100644 index 0000000..da8a53e --- /dev/null +++ b/testing/Baseline/tests.trace/conn.log.filtered @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 172.22.214.60 8 192.0.78.150 0 icmp - - - - OTH 0 - 1 28 0 0 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 172.22.214.60 8 192.0.78.212 0 icmp - - - - OTH 0 - 1 28 0 0 - diff --git a/testing/Baseline/tests.trace/output b/testing/Baseline/tests.trace/output new file mode 100644 index 0000000..018b8d7 --- /dev/null +++ b/testing/Baseline/tests.trace/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Testing protobuf: [c8:5b:76:bd:77:ab -> ff:ff:ff:ff:ff:ff] I am encapsulating diff --git a/testing/Files/random.seed b/testing/Files/random.seed new file mode 100644 index 0000000..6956a2c --- /dev/null +++ b/testing/Files/random.seed @@ -0,0 +1,21 @@ +2983378351 +1299727368 +0 +310447 +0 +1409073626 +3975311262 +34130240 +1450515018 +1466150520 +1342286698 +1193956778 +2188527278 +3361989254 +3912865238 +3596260151 +517973768 +1462428821 +0 +2278350848 +32767 diff --git a/testing/Makefile b/testing/Makefile new file mode 100644 index 0000000..b43cce5 --- /dev/null +++ b/testing/Makefile @@ -0,0 +1,3 @@ + +test: + @btest -c btest.cfg diff --git a/testing/Scripts/README b/testing/Scripts/README new file mode 100644 index 0000000..8098fa4 --- /dev/null +++ b/testing/Scripts/README @@ -0,0 +1,7 @@ +Place helper scripts, such a btest-diff canonifiers, in this directory. +Note that Zeek versions 4.1 and newer include their btest tooling as part +of the installation. Take a look at the folder reported via + + zeek-config --btest_tools_dir + +for scripts, PRNG seeds, and pcaps you might be able to reuse. diff --git a/testing/Scripts/diff-remove-timestamps b/testing/Scripts/diff-remove-timestamps new file mode 100755 index 0000000..8ab13f5 --- /dev/null +++ b/testing/Scripts/diff-remove-timestamps @@ -0,0 +1,13 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). + +# Get us "modern" regexps with sed. +if [ "$(uname)" == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | + $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/testing/Scripts/get-zeek-env b/testing/Scripts/get-zeek-env new file mode 100755 index 0000000..d92cb9e --- /dev/null +++ b/testing/Scripts/get-zeek-env @@ -0,0 +1,40 @@ +#! /bin/sh +# +# BTest helper for getting values for Zeek-related environment variables. + +# shellcheck disable=SC2002 +base="$(dirname "$0")" +zeek_dist=$(cat "${base}/../../build/CMakeCache.txt" 2>/dev/null | grep ZEEK_DIST | cut -d = -f 2) + +if [ -n "${zeek_dist}" ]; then + if [ "$1" = "zeekpath" ]; then + "${zeek_dist}/build/zeek-path-dev" + elif [ "$1" = "zeek_plugin_path" ]; then + (cd "${base}/../.." && pwd) + elif [ "$1" = "path" ]; then + echo "${zeek_dist}/build/src:${zeek_dist}/aux/btest:${base}/:${zeek_dist}/aux/zeek-cut:$PATH" + else + echo "usage: $(basename "$0") " >&2 + exit 1 + fi +else + # Use Zeek installation for testing. In this case zeek-config must be in PATH. + if ! which zeek-config >/dev/null 2>&1; then + echo "zeek-config not found" >&2 + exit 1 + fi + + if [ "$1" = "zeekpath" ]; then + zeek-config --zeekpath + elif [ "$1" = "zeek_plugin_path" ]; then + # Combine the local tree and the system-wide path. This allows + # us to test on a local build or an installation made via zkg, + # which squirrels away the build. + echo "$(cd "${base}/../.." && pwd)/build:$(zeek-config --plugin_dir)" + elif [ "$1" = "path" ]; then + echo "${PATH}" + else + echo "usage: $(basename "$0") " >&2 + exit 1 + fi +fi diff --git a/testing/Traces/raw-layer.pcap b/testing/Traces/raw-layer.pcap new file mode 100644 index 0000000000000000000000000000000000000000..5e4ed891dca385beeec0a58c85689b3790b843ed GIT binary patch literal 161 zcmca|c+)~A1{MYw`2U}Qff2|FzvUmFQN+oh1!RNpe=s-^UADJ;H3x$$1A`1mnFHg? z9c#p{*&JZ-yTZZn9jF(gv+p4%gFTQ9!pJ&1wt6Zg<|?G-B_|dXm*ymvWagzScm^OD L_YG>?G?;M!%daoutput +# +# Zeek 6 and newer populate the local_orig and local_resp columns by default, +# while earlier ones only do so after manual configuration. Filter out these +# columns to allow robust baseline comparison: +# @TEST-EXEC: cat conn.log | zeek-cut -m -n local_orig local_resp >conn.log.filtered +# +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff conn.log.filtered +# +# @TEST-DOC: Test Zeek parsing a trace file through the protobuf analyzer. + +# TODO: Adapt as suitable. The example only checks the output of the event +# handler. + +event protobuf::packet(p: raw_pkt_hdr, payload: string) + { + print fmt("Testing protobuf: [%s -> %s] %s", p$l2$src, p$l2$dst, payload); + } diff --git a/zkg.meta b/zkg.meta new file mode 100644 index 0000000..8535022 --- /dev/null +++ b/zkg.meta @@ -0,0 +1,23 @@ +[package] +script_dir = scripts +test_command = cd testing && btest -c btest.cfg +summary = TODO: A summary of spicy-protobuf in one line +description = TODO: A more detailed description of spicy-protobuf. + It can span multiple lines, with this indentation. +depends = + zeek >=4.0.0 +build_command = mkdir -p build && cd build && SPICYZ=$(command -v spicyz || echo %(package_base)s/spicy-plugin/build/bin/spicyz) cmake .. && cmake --build . + +[template] +source = https://github.com/zeek/package-template +version = v3.1.0 +zkg_version = 2.14.0 +features = github-ci,license,spicy-packet-analyzer + +[template_vars] +name = spicy-protobuf +analyzer = protobuf +unit = Packet +author = Tanner Kvarfordt tanner.kvarfordt@gmail.com +license = mit +