fix: filter fortigate (#2579)

splunk · Sep 26, 2024 · 7dec478 · 7dec478
1 parent 5e9cf16
commit 7dec478
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 2 deletions.
diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortios.conf b/package/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortios.conf
@@ -95,6 +95,14 @@ block parser app-syslog-fortigate_fortios() {
         };
    };
 };
+
 application fortinet_fortios[sc4s-syslog] {
+    filter {
+        (
+            message("eventtype=")
+            or message("time=")
+        )
+        and message("devid=")
+    };
 	parser { app-syslog-fortigate_fortios(); };
 };
diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortiweb.conf b/package/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortiweb.conf
@@ -133,5 +133,15 @@ block parser app-syslog-fortigate_fortiweb() {
    };
 };
 application app-syslog-fortigate_fortiweb[sc4s-syslog] {
+    filter {
+        (
+            message('eventtime=')
+            or (
+                message('date=')
+                and message('time=')
+            )
+        )
+        and message('device_id=')
+    };
 	parser { app-syslog-fortigate_fortiweb(); };
 };
diff --git a/package/lite/etc/addons/fortinet/app-syslog-fortigate_fortios.conf b/package/lite/etc/addons/fortinet/app-syslog-fortigate_fortios.conf
@@ -95,6 +95,14 @@ block parser app-syslog-fortigate_fortios() {
         };
    };
 };
+
 application fortinet_fortios[sc4s-syslog] {
+    filter {
+        (
+            message("eventtype=")
+            or message("time=")
+        )
+        and message("devid=")
+    };
 	parser { app-syslog-fortigate_fortios(); };
 };
diff --git a/package/lite/etc/addons/fortinet/app-syslog-fortigate_fortiweb.conf b/package/lite/etc/addons/fortinet/app-syslog-fortigate_fortiweb.conf
@@ -133,5 +133,15 @@ block parser app-syslog-fortigate_fortiweb() {
    };
 };
 application app-syslog-fortigate_fortiweb[sc4s-syslog] {
+    filter {
+        (
+            message('eventtime=')
+            or (
+                message('date=')
+                and message('time=')
+            )
+        )
+        and message('device_id=')
+    };
 	parser { app-syslog-fortigate_fortiweb(); };
 };
diff --git a/tests/splunkutils.py b/tests/splunkutils.py
@@ -6,7 +6,7 @@
 from time import sleep
 
 
-def splunk_single(service, search):
+def splunk_single(service, search, attempt_limit=10):
     kwargs_normalsearch = {"exec_mode": "normal"}
     tried = 0
     while True:
@@ -32,7 +32,7 @@ def splunk_single(service, search):
         # Get the results and display them
         result_count = stats["resultCount"]
         event_count = stats["eventCount"]
-        if result_count > 0 or tried > 10:
+        if result_count > 0 or tried > attempt_limit:
             break
         else:
             tried += 1

diff --git a/tests/test_fortinet_ngfw.py b/tests/test_fortinet_ngfw.py
@@ -270,3 +270,36 @@ def test_fortinet_fgt_event_et_epochms(record_property,  setup_splunk, setup_sc4
     record_property("message", message)
 
     assert result_count == 1
+
+# Check a sample message that is not Fortinet, but without proper filters, it passes through the Fortinet's kv-parser 
+# and triggers the warning 'Value names cannot be longer than 255 characters, this value will always expand to the empty string.' 
+# See https://github.com/splunk/splunk-connect-for-syslog/issues/2297
+
+# <13>Nov 08 12:59:54 1.1.1.1 program[-]: VERSION:v1:date_time='2023-11-08 13:59:54',clientip='1.2.2.2',host='[host.example.com](https://host.example.com/)' ,http_host='[host.example.com](https://host.example.com/)',http_responsecode='200',http_username='makemelongenoughtotriggerAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABASE64CONTENTendingwitha=',http_user-agent='PHP-SOAP-CURL',http_referer='',http_xff='3.3.3.3',http_request_id='',cached='false',virtualname='something',virtualip='4.4.4.4',virtualport='443',http_method='POST',http_path='/bla/blub.asmx',http_query='',http_version='HTTP/1.1',http_response_size='10092',http_response_time='32',nodeip='4.4.4.4',nodeport='443',snatpool='/Common/SNAT_Something_Pool',snatip='6.6.6.6',snatport='34470',pool='/Common/blub.app/blapool8',req_type='response'
+@pytest.mark.addons("fortinet")
+def test_fortinet_prefiltering(record_property,  setup_splunk, setup_sc4s):
+    dt = datetime.datetime.now()
+    _, bsd, _, _, _, _, _ = time_operations(dt)
+
+    unique_substring = f"{shortuuid.ShortUUID().random(length=5).lower()}{shortuuid.ShortUUID().random(length=5).lower()}"
+
+    mt = env.from_string(
+        "{{ mark }} {{ bsd }} 1.1.1.1 program[-]: VERSION:v1:date_time='2023-11-08 13:59:54',clientip='1.2.2.2',host='[host.example.com](https://host.example.com/)' ,http_host='[host.example.com](https://host.example.com/)',http_responsecode='200',http_username='makemelongenoughtotrigger{{ unique_substring }}AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABASE64CONTENTendingwitha=',http_user-agent='PHP-SOAP-CURL',http_referer='',http_xff='3.3.3.3',http_request_id='',cached='false',virtualname='something',virtualip='4.4.4.4',virtualport='443',http_method='POST',http_path='/bla/blub.asmx',http_query='',http_version='HTTP/1.1',http_response_size='10092',http_response_time='32',nodeip='4.4.4.4',nodeport='443',snatpool='/Common/SNAT_Something_Pool',snatip='6.6.6.6',snatport='34470',pool='/Common/blub.app/blapool8',req_type='response'"
+    )
+    message = mt.render(
+        mark="<13>", bsd=bsd, unique_substring=unique_substring
+    )
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search index=main sourcetype="sc4s:events" | search "Value names cannot be longer than 255 characters" value="*{{ unique_substring }}*"'
+    )
+    search = st.render(unique_substring=unique_substring)
+
+    result_count, _ = splunk_single(setup_splunk, search, 2)
+
+    record_property("resultCount", result_count)
+    record_property("message", message)
+
+    assert result_count == 0