Update for ProofPoint pod log to include mime types

config/processors/api_security_proofpoint.pod_email_gateway.conf

@@ -6,6 +6,9 @@ input {
   }
 }  
 filter {
+  mutate{  
+    remove_field => [ "host", "event" ]   
+  }
   mutate {
     copy => {"message" => "tmp_msg"}
     add_field => { "[cloud][provider]" => "proofpoint" }
@@ -22,11 +25,51 @@ filter {
     source => "tmp_msg"
     target => "pp" 
   }
+  # lowercase field names
+  ruby {
+    init => '@ignore = [ "path", "@timestamp", "@metadata", "host", "@version" ]'
+    code => '
+      def processArray(a)
+        newArray = []
+        a.each { |x|
+          newArray << processObject(x)
+        }
+        newArray
+      end
+      def processHash(h)
+        newHash = {}
+        h.each { |k, v|
+          newHash[k.downcase] = processObject(v)
+        }
+        newHash
+      end
+      def processObject(v)
+        if v.kind_of?(Array)
+          processArray(v)
+        elsif v.kind_of?(Hash)
+          processHash(v)
+        else
+          v
+        end
+      end
+      def filter(i_event)
+        i_event.to_hash.each { |k, v|
+          unless @ignore.include?(k)
+            i_event.remove(k)
+            i_event.set(k.downcase, processObject(v))
+          end
+        }
+        [i_event]
+      end
+      filter(event)
+    '
+  }
+# [file][mime_type]
   mutate {
-    rename => { "[pp][msg][parsedAddresses][from]" => "[source][user][email]" }
+    rename => { "[pp][msg][parsedaddresses][from]" => "[source][user][email]" }
     rename => { "[pp][envelope][rcpts]" => "[destination][user][email]" }
-    merge => { "[destination][user][email]" => "[pp][msg][parsedAddresses][cc]" }
-    rename => { "[pp][msg][normalizedHeader][subject]" => "[email][subject]" }
+    merge => { "[destination][user][email]" => "[pp][msg][parsedaddresses][cc]" }
+    rename => { "[pp][msg][normalizedheader][subject]" => "[email][subject]" }
     rename => { "[pp][ts]" => "[event][created]" }
     rename => { "[pp][guid]" => "[rule][uuid]" }
     rename => { "[pp][pps][cid]" => "[observer][name]" }
@@ -35,9 +78,9 @@ filter {
     rename => { "[pp][filter][disposition]" => "[event][outcome]" }
     rename => { "[pp][filter][quarantine][folder]" => "[file][path]" }
     rename => { "[pp][filter][quarantine][rule]" => "[rule][name]" }
-    rename => { "[pp][filter][routeDirection]" => "[network][direction]" }
-    rename => { "[pp][filter][startTime]" => "[event][start]"}
-    rename => { "[pp][filter][msgSizeBytes]" => "[source][bytes]"}
+    rename => { "[pp][filter][routedirection]" => "[network][direction]" }
+    rename => { "[pp][filter][starttime]" => "[event][start]"}
+    rename => { "[pp][filter][msgsizebytes]" => "[source][bytes]"}
     rename => { "[pp][connection][ip]" => "[source][ip]" }
     rename => { "[pp][connection][host]" => "[source][address]" }
     rename => { "[pp][filter][actions]" => "[proofpoint][outcome]" }
@@ -49,17 +92,28 @@ filter {
   ruby {
     code => "
       detected_name_list = []
+      label_mime_list = []
+      detected_mime_list = []
       detected_size_bytes_list = []
       md5_list = []
       sha256_list = []
       urls_list = []
-      if event.get('[pp][msgParts]').kind_of?(Array)
-        event.get('[pp][msgParts]').each do |item|
-          if item['detectedName'] != ''
-            detected_name_list.push(item['detectedName'])
+      if event.get('[pp][msgparts]').kind_of?(Array)
+        event.get('[pp][msgparts]').each do |item|
+          if item['detectedname'] != ''
+            detected_name_list.push(item['detectedname'])
+          end
+
+          if item['labeledmime'] != ''
+            label_mime_list.push(item['labeledmime'])
+          end
+          if item['detectedmime'] != ''
+            detected_mime_list.push(item['detectedmime'])
           end
-          if item['detectedSizeBytes'] != ''
-            detected_size_bytes_list.push(item['detectedSizeBytes'])
+
+
+          if item['detectedsizebytes'] != ''
+            detected_size_bytes_list.push(item['detectedsizebytes'])
           end
           if item['sha256'] != ''
             sha256_list.push(item['sha256'])
@@ -79,6 +133,14 @@ filter {
       if !detected_name_list.empty?
         event.set('[file][name]', detected_name_list)
       end
+
+      if !label_mime_list.empty?
+        event.set('[file][mime_type]', label_mime_list)
+      end
+      if !detected_mime_list.empty?
+        event.set('[file][detected_mime_type]', detected_mime_list)
+      end
+
       if !detected_size_bytes_list.empty?
         event.set('[file][size]', detected_size_bytes_list)
       end
@@ -94,9 +156,9 @@ filter {
     "
     tag_on_exception => "msgparts_ruby_block"
   }
-  if [pp][msg][parsedAddresses][to][0] !~ '^.*Undisclosed.*$' {
+  if [pp][msg][parsedaddresses][to][0] !~ '^.*undisclosed.*$' {
     mutate {
-      merge => { "[destination][user][email]" => "[pp][msg][parsedAddresses][to]" }
+      merge => { "[destination][user][email]" => "[pp][msg][parsedaddresses][to]" }
     }
   }
   if [pp][connection][protocol] {
@@ -116,6 +178,33 @@ filter {
       gsub => [ "[source][address]", "[\[\]]", "" ]
     }
   }
+  # file count and tottal
+  ruby {
+    code => '
+     event.set("[file][count]", event.get("[file][name]").length)
+     event.set("[file][size_total]", event.get("[file][size]").sum)
+    '
+  }
+  ruby {
+    code => '
+      dmt = event.get("[file][detected_mime_type]").to_a
+      fl = event.get("[file][name]").to_a
+      mt = event.get("[file][mime_type]").to_a
+
+      if mt.length != dmt.length or mt.length != fl.length
+        event.set("tags", event.get("tags").to_a.append("_file_array_size_mismatch"))
+        return
+      end
+
+      mt.each_with_index do |val, i|
+        if val != dmt[i]
+          event.set("tags", event.get("tags").to_a.append("_mime_type_mismatch__"+fl[i]))
+        end
+      end
+    '
+  }
+
+
   # "ts":"2024-07-24t13:52:24.333621-0500"
   date {
     match => ["[event][created]", "ISO8601"]
@@ -129,8 +218,11 @@ filter {
     target => "[event][start]"
     tag_on_failure => "_dateparsefailure_es"
   }
+  grok {
+    match => { "[file][name]" => "^.*\.(?<[file][extension]>.*?$)" }
+  }
   mutate {
-    remove_field => ["tmp_msg", "pp"]
+    remove_field => ["tmp_msg", "pp" ]
   }
 }
 output {