diff --git a/config/processors/api_security_proofpoint.pod_email_gateway.conf b/config/processors/api_security_proofpoint.pod_email_gateway.conf index 641ac9d5..adb8945c 100644 --- a/config/processors/api_security_proofpoint.pod_email_gateway.conf +++ b/config/processors/api_security_proofpoint.pod_email_gateway.conf @@ -6,6 +6,9 @@ input { } } filter { + mutate{ + remove_field => [ "host", "event" ] + } mutate { copy => {"message" => "tmp_msg"} add_field => { "[cloud][provider]" => "proofpoint" } @@ -22,11 +25,51 @@ filter { source => "tmp_msg" target => "pp" } + # lowercase field names + ruby { + init => '@ignore = [ "path", "@timestamp", "@metadata", "host", "@version" ]' + code => ' + def processArray(a) + newArray = [] + a.each { |x| + newArray << processObject(x) + } + newArray + end + def processHash(h) + newHash = {} + h.each { |k, v| + newHash[k.downcase] = processObject(v) + } + newHash + end + def processObject(v) + if v.kind_of?(Array) + processArray(v) + elsif v.kind_of?(Hash) + processHash(v) + else + v + end + end + def filter(i_event) + i_event.to_hash.each { |k, v| + unless @ignore.include?(k) + i_event.remove(k) + i_event.set(k.downcase, processObject(v)) + end + } + [i_event] + end + filter(event) + ' + } +# [file][mime_type] mutate { - rename => { "[pp][msg][parsedAddresses][from]" => "[source][user][email]" } + rename => { "[pp][msg][parsedaddresses][from]" => "[source][user][email]" } rename => { "[pp][envelope][rcpts]" => "[destination][user][email]" } - merge => { "[destination][user][email]" => "[pp][msg][parsedAddresses][cc]" } - rename => { "[pp][msg][normalizedHeader][subject]" => "[email][subject]" } + merge => { "[destination][user][email]" => "[pp][msg][parsedaddresses][cc]" } + rename => { "[pp][msg][normalizedheader][subject]" => "[email][subject]" } rename => { "[pp][ts]" => "[event][created]" } rename => { "[pp][guid]" => "[rule][uuid]" } rename => { "[pp][pps][cid]" => "[observer][name]" } @@ -35,9 +78,9 @@ filter { rename => { "[pp][filter][disposition]" => "[event][outcome]" } rename => { "[pp][filter][quarantine][folder]" => "[file][path]" } rename => { "[pp][filter][quarantine][rule]" => "[rule][name]" } - rename => { "[pp][filter][routeDirection]" => "[network][direction]" } - rename => { "[pp][filter][startTime]" => "[event][start]"} - rename => { "[pp][filter][msgSizeBytes]" => "[source][bytes]"} + rename => { "[pp][filter][routedirection]" => "[network][direction]" } + rename => { "[pp][filter][starttime]" => "[event][start]"} + rename => { "[pp][filter][msgsizebytes]" => "[source][bytes]"} rename => { "[pp][connection][ip]" => "[source][ip]" } rename => { "[pp][connection][host]" => "[source][address]" } rename => { "[pp][filter][actions]" => "[proofpoint][outcome]" } @@ -49,17 +92,28 @@ filter { ruby { code => " detected_name_list = [] + label_mime_list = [] + detected_mime_list = [] detected_size_bytes_list = [] md5_list = [] sha256_list = [] urls_list = [] - if event.get('[pp][msgParts]').kind_of?(Array) - event.get('[pp][msgParts]').each do |item| - if item['detectedName'] != '' - detected_name_list.push(item['detectedName']) + if event.get('[pp][msgparts]').kind_of?(Array) + event.get('[pp][msgparts]').each do |item| + if item['detectedname'] != '' + detected_name_list.push(item['detectedname']) + end + + if item['labeledmime'] != '' + label_mime_list.push(item['labeledmime']) + end + if item['detectedmime'] != '' + detected_mime_list.push(item['detectedmime']) end - if item['detectedSizeBytes'] != '' - detected_size_bytes_list.push(item['detectedSizeBytes']) + + + if item['detectedsizebytes'] != '' + detected_size_bytes_list.push(item['detectedsizebytes']) end if item['sha256'] != '' sha256_list.push(item['sha256']) @@ -79,6 +133,14 @@ filter { if !detected_name_list.empty? event.set('[file][name]', detected_name_list) end + + if !label_mime_list.empty? + event.set('[file][mime_type]', label_mime_list) + end + if !detected_mime_list.empty? + event.set('[file][detected_mime_type]', detected_mime_list) + end + if !detected_size_bytes_list.empty? event.set('[file][size]', detected_size_bytes_list) end @@ -94,9 +156,9 @@ filter { " tag_on_exception => "msgparts_ruby_block" } - if [pp][msg][parsedAddresses][to][0] !~ '^.*Undisclosed.*$' { + if [pp][msg][parsedaddresses][to][0] !~ '^.*undisclosed.*$' { mutate { - merge => { "[destination][user][email]" => "[pp][msg][parsedAddresses][to]" } + merge => { "[destination][user][email]" => "[pp][msg][parsedaddresses][to]" } } } if [pp][connection][protocol] { @@ -116,6 +178,33 @@ filter { gsub => [ "[source][address]", "[\[\]]", "" ] } } + # file count and tottal + ruby { + code => ' + event.set("[file][count]", event.get("[file][name]").length) + event.set("[file][size_total]", event.get("[file][size]").sum) + ' + } + ruby { + code => ' + dmt = event.get("[file][detected_mime_type]").to_a + fl = event.get("[file][name]").to_a + mt = event.get("[file][mime_type]").to_a + + if mt.length != dmt.length or mt.length != fl.length + event.set("tags", event.get("tags").to_a.append("_file_array_size_mismatch")) + return + end + + mt.each_with_index do |val, i| + if val != dmt[i] + event.set("tags", event.get("tags").to_a.append("_mime_type_mismatch__"+fl[i])) + end + end + ' + } + + # "ts":"2024-07-24t13:52:24.333621-0500" date { match => ["[event][created]", "ISO8601"] @@ -129,8 +218,11 @@ filter { target => "[event][start]" tag_on_failure => "_dateparsefailure_es" } + grok { + match => { "[file][name]" => "^.*\.(?<[file][extension]>.*?$)" } + } mutate { - remove_field => ["tmp_msg", "pp"] + remove_field => ["tmp_msg", "pp" ] } } output {