Fix/cisco meraki update

docs/sources/vendor/Cisco/cisco_meraki.md

@@ -1,29 +1,49 @@
-## Meraki (MR, MS, MX, MV)
+## Meraki (MR, MS, MX)
 
 ## Key facts
+* In most cases, Cisco Meraki logs are general and require vendor product by source configuration.
+* For distinctive log messages, filters are based on the appliance name and program value.
 
-* MSG Format based filter (Partial)
-* Requires vendor product by source configuration
-* None conformant legacy BSD Format default port 514
+## Distinctive log messages
+See samples in the [vendor documentation](https://documentation.meraki.com/General_Administration/Monitoring_and_Reporting/Syslog_Event_Types_and_Log_Samples).
+
+The two conjuncted conditions are required:
+
+1. Program: `(events|urls|firewall|cellular_firewall|vpn_firewall|ids-alerts|flows)`
+
+2. Appliance name:
+
+| Sourcetype | Distinct element |
+| ---------  | --------------   |
+| meraki:accesspoints | `host('MR' type(string) flags(ignore-case,prefix))` |
+| meraki:securityappliances | `host('MX' type(string) flags(ignore-case,prefix))` |
+| meraki:switches | `host('MS' type(string) flags(ignore-case,prefix))` |
+
 
 ## Links
 
 | Ref            | Link                                                                                                    |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| Splunk Add-on  | <https://splunkbase.splunk.com/app/3018/>                                                                 |
-| Product Manual | <https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration> |
+| Splunk Add-on  | <https://splunkbase.splunk.com/app/5580>                                                                 |
+| Product Manual | <https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration> <https://documentation.meraki.com/General_Administration/Monitoring_and_Reporting/Syslog_Event_Types_and_Log_Samples> |
 
 ## Sourcetypes
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| meraki     | None                                                                                                    |
+| meraki:accesspoints    | MR                                                                                                |
+| meraki:securityappliances    | MX                                                                                              |
+| meraki:switches    | MS                                                                                             |
+| meraki | vendor product by source configuration |
 
 ## Sourcetype and Index Configuration
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| cisco_meraki     | meraki    | netfw          | The current TA does not sub sourcetype or utilize source preventing segmentation into more appropriate indexes |
+| cisco_meraki_accesspoints     | meraki:accesspoints    | netfw          | Filtered on the message format |
+| cisco_meraki_securityappliances     | meraki:securityappliances    | netfw          | Filtered on the message format |
+| cisco_meraki_switches     | meraki:switches    | netfw          | Filtered on the message format |
+| cisco_meraki | meraki | netfw | Filtered on vendor product by source configuration |
 
 ## Parser Configuration
 

package/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_meraki.conf

@@ -0,0 +1,28 @@
+rewrite set_rfc3164_cisco_meraki{
+    set-tag("wireformat:rfc3164_cisco_meraki");
+};
+
+block parser app-almost-syslog-cisco_meraki() {
+    channel {
+        parser {
+            regexp-parser(
+                prefix(".tmp.")
+                patterns('^(?<pri>\<\d+\>) ?(?<ts1>[A-Z][a-z]{2} *\d{1,2} \d\d:\d\d:\d\d) (?<sender>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (?<pris>\d) (?<ts2>\d{10}\.\d{9}) (?<message>.*)')
+            );
+            date-parser(
+                format('%s.%f', '%s')
+                template("${.tmp.ts2}")
+            );
+            syslog-parser(
+                flags(assume-utf8, guess-timezone)
+                template("${.tmp.pri} $S_ISODATE ${.tmp.message}")
+            );
+        };
+        rewrite(set_rfc);
+        rewrite(set_rfc3164);
+        rewrite(set_rfc3164_cisco_meraki);
+    };
+};
+application app-almost-syslog-cisco_meraki[sc4s-almost-syslog] {
+    parser { app-almost-syslog-cisco_meraki(); };
+};

package/etc/conf.d/conflib/syslog/app-syslog-cisco_meraki.conf

@@ -0,0 +1,45 @@
+block parser app-syslog-cisco_meraki() {
+    channel {
+
+        rewrite {
+            set("securityappliances", value(".tmp.device") condition( host('MX' type(string) flags(ignore-case,prefix))));
+            set("switches", value(".tmp.device") condition( host('MS' type(string) flags(ignore-case,prefix))));
+            set("accesspoints", value(".tmp.device") condition( host('MR' type(string) flags(ignore-case,prefix))));
+        };
+
+        rewrite {
+            r_set_splunk_dest_default(
+                index('netfw')
+                source('cisco:meraki:${.tmp.device}')
+                sourcetype('meraki:${.tmp.device}')
+                vendor("cisco")
+                product("meraki")
+                class("${.tmp.device}")
+            );
+        };
+    };
+};
+
+application app-syslog-cisco_meraki[sc4s-syslog] {
+    filter {
+        (
+            (
+                host('MX' type(string) flags(ignore-case,prefix))
+                or host('MS' type(string) flags(ignore-case,prefix))
+                or host('MR' type(string) flags(ignore-case,prefix))
+            )
+        )
+        and (
+            (
+                program('events' type(string))
+                or program('urls' type(string))
+                or program('firewall' type(string))
+                or program('cellular_firewall' type(string))
+                or program('vpn_firewall' type(string))
+                or program('ids-alerts' type(string))
+                or program('flows' type(string))
+            )
+        )
+    };
+    parser { app-syslog-cisco_meraki() };
+};

tests/test_cisco_meraki.py

@@ -11,10 +11,145 @@
 from .splunkutils import *
 from .timeutils import *
 
+import pytest
+
 env = Environment()
 
+# Log samples from https://documentation.meraki.com/General_Administration/Monitoring_and_Reporting/Syslog_Event_Types_and_Log_Samples
+mx_test_data = [
+    # MX events: vpn connectivity change
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events type=vpn_connectivity_change vpn_type='site-to-site' peer_contact='1.1.1.1:51856' peer_ident='XXXXX' connectivity='false'",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # urls: HTTP GET requests
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} urls src=1.1.1.1:63735 dst=1.1.1.1:80 mac=XX:XX:XX:XX:XX:XX request: GET https://...",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # MX flows
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} flows src=1.1.1.186 dst=8.8.8.8 mac=XX:XX:XX:XX:XX:XX protocol=udp sport=55719 dport=53 pattern: allow all",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # MX firewall
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} firewall src=1.1.1.186 dst=8.8.8.8 mac=XX:XX:XX:XX:XX:XX protocol=udp sport=55719 dport=53 pattern: allow all",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # MX ids-alerts: ids signature matched
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} ids-alerts signature=129:4:1 priority=3 timestamp=1377449842.512569 direction=ingress protocol=tcp/ip src=1.1.1.1:80",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    }
+]
+
+ms_test_data = [
+    # MS events: port status change
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events port 3 status changed from 100fdx to down",
+        "host_prefix": "MS",
+        "sourcetype": "meraki:switches"
+    },
+    # MS events: blocked DHCP server response
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events Blocked DHCP server response from XX:XX:XX:XX:XX:XX on VLAN 100",
+        "host_prefix": "MS",
+        "sourcetype": "meraki:switches"
+    }
+]
+
+mr_test_data = [
+    # MR events: 802.11 association
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events type=association radio='0' vap='1' channel='6' rssi='23' aid='XXXXXX'",
+        "host_prefix": "MR",
+        "sourcetype": "meraki:accesspoints"
+    },
+    # MR events: WPA authentication
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events type=wpa_auth radio='0' vap='1' aid='XXXXXXX'",
+        "host_prefix": "MR",
+        "sourcetype": "meraki:accesspoints"
+    },
+    # MR events: splash authentication
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} events type=splash_auth ip='1.1.1.1 [More Information] ' duration='3600' vap='2' download='5242880bps' upload='5242880bps'",
+        "host_prefix": "MR",
+        "sourcetype": "meraki:accesspoints"
+    },
+    # MR flows: flow denied by Layer 3 firewall
+    {
+        "template": "{{ mark }} {{ epoch }} {{ host }} flows deny src=1.1.1.1 dst=1.1.1.1 mac=XX:XX:XX:XX:XX:XX protocol=tcp sport=52421 dport=80",
+        "host_prefix": "MR",
+        "sourcetype": "meraki:accesspoints"
+    }
+]
+
+mx_almost_syslog_test_data = [
+    # MX events: uplink connectivity change
+    {
+        "template": "{{ mark }} Dec 6 08:46:12 1.1.1.1 1 {{ epoch }} {{ host }} events Cellular connection down",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # MX events: dhcp no offers
+    {
+        "template": "{{ mark }} Sep 11 16:12:41 1.1.1.1 1 {{ epoch }} {{ host }} events dhcp no offers for mac XX:XX:XX:XX:XX:XX host = 1.1.1.1",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    },
+    # MX events: dhcp lease
+    {
+        "template": "{{ mark }} Sep 11 16:05:15 1.1.1.1 1 {{ epoch }} {{ host }} events dhcp lease of ip 1.1.1.1 from server mac XX:XX:XX:XX:XX:XX for client mac XX:XX:XX:XX:XX:XX from router 1.1.1.1 on subnet 255.255.255.0 with dns 8.8.8.8, 8.8.4.4",
+        "host_prefix": "MX",
+        "sourcetype": "meraki:securityappliances"
+    }
+]
+
+test_data = mx_test_data + ms_test_data + mr_test_data + mx_almost_syslog_test_data
+
+
+@pytest.mark.parametrize("test_case", test_data)
+def test_cisco_meraki_syslog_app(
+        record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, test_case
+):
+    model_number = random.randint(60, 200)
+    model_suffix = random.choice(["", "C", "CW", "W", "-HW", "W-HW"])
+    host = f'{test_case["host_prefix"]}{model_number}{model_suffix}'
+
+    dt = datetime.datetime.now(datetime.timezone.utc)
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    meraki_format_epoch = epoch + "000" # "1691740392.147501" -> "1691740392.147501000"
+
+    mt = env.from_string(test_case["template"] + "\n")
+    message = mt.render(mark="<134>", epoch=meraki_format_epoch, host=host)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    epoch = dt.astimezone().strftime("%s.%f")[:-3] # -> "1691740392.147501" -> "1691740392.147"
+    st = env.from_string(
+        'search index=netfw _time={{ epoch }} sourcetype={{ sourcetype }} host={{ host }}'
+    )
+    search = st.render( epoch=epoch, sourcetype=test_case["sourcetype"], host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1
+
+
 # <134>1 1563249630.774247467 devicename security_event ids_alerted signature=1:28423:1 priority=1 timestamp=1468531589.810079 dhost=98:5A:EB:E1:81:2F direction=ingress protocol=tcp/ip src=151.101.52.238:80 dst=192.168.128.2:53023 message: EXPLOIT-KIT Multiple exploit kit single digit exe detection
-def test_cisco_meraki_security_event(
+def test_cisco_meraki_vps_app(
     record_property, setup_wordlist, setup_splunk, setup_sc4s
 ):
     host = "testcm-{}-{}".format(
@@ -45,7 +180,4 @@ def test_cisco_meraki_security_event(
     record_property("resultCount", resultCount)
     record_property("message", message)
 
-    assert resultCount == 1
-
-
-# <134>1 Dec 6 08:41:44 192.168.1.1 1 1386337316.207232138 MX84 events Cellular connection up
+    assert resultCount == 1