From 7c69acf71f1356b8e02a376656616f3f862af30e Mon Sep 17 00:00:00 2001
From: Daryl Coburn <daryl_coburn@cargill.com>
Date: Fri, 25 Oct 2024 09:03:39 -0600
Subject: [PATCH] removed 96_ecs, rnmd output to 96

---
 config/enrichments/96_ecs_data_type.conf      | 321 ------------------
 .../{97_output.conf => 96_output.conf}        |   0
 2 files changed, 321 deletions(-)
 delete mode 100644 config/enrichments/96_ecs_data_type.conf
 rename config/enrichments/{97_output.conf => 96_output.conf} (100%)

diff --git a/config/enrichments/96_ecs_data_type.conf b/config/enrichments/96_ecs_data_type.conf
deleted file mode 100644
index 54093697..00000000
--- a/config/enrichments/96_ecs_data_type.conf
+++ /dev/null
@@ -1,321 +0,0 @@
-# Copyright [2021] [Cargill, Incorporated.] 
-# SPDX-License-Identifier: Apache-2.0
-# remove empty fields and truncate all fields to 1024 bytes
-filter {
-    mutate {
-      rename => { "log.source.ip" => "[log][source][ip]" }
-      rename => { "log.source.hostname" => "[log][source][hostname]" }
-    }
-  if "disable_ecs_data_type_enrichment" in [tags] {
-    mutate {
-      remove_tag => ["disable_ecs_data_type_enrichment"]
-    }
-  } else {
-    # Remove custom shipper topic name
-    mutate {
-      remove_field => ["log_source_type"]
-    }
-    # remove fields which are not convertible to ecs datatypes
-    ruby {
-      code => '
-        long_fields = [
-          "[client][bytes]",
-          "[client][packets]",
-          "[client][port]",
-          "[destination][bytes]",
-          "[destination][packets]",
-          "[destination][port]",
-          "[event][duration]",
-          "[event][severity]",
-          "[file][x509][public_key_exponent]",
-          "[network][bytes]",
-          "[network][packets]",
-          "[process][parent][pid]",
-          "[process][pid]",
-          "[server][bytes]",
-          "[server][packets]",
-          "[server][port]",
-          "[source][bytes]",
-          "[source][packets]",
-          "[source][port]",
-          "[client][as][number]",
-          "[client][nat][port]",
-          "[destination][as][number]",
-          "[destination][nat][port]",
-          "[dns][answers][ttl]",
-          "[event][sequence]",
-          "[file][size]",
-          "[file][x509][public_key_size]",
-          "[host][disk][read][bytes]",
-          "[host][disk][write][bytes]",
-          "[host][network][egress][bytes]",
-          "[host][network][egress][packets]",
-          "[host][network][ingress][bytes]",
-          "[host][network][ingress][packets]",
-          "[host][uptime]",
-          "[http][request][body][bytes]",
-          "[http][request][bytes]",
-          "[http][response][body][bytes]",
-          "[http][response][bytes]",
-          "[http][response][status_code]",
-          "[log][origin][file][line]",
-          "[log][syslog][facility][code]",
-          "[log][syslog][priority]",
-          "[log][syslog][severity][code]",
-          "[observer][ingress][interface][host_count]",
-          "[package][size]",
-          "[process][args_count]",
-          "[process][exit_code]",
-          "[process][parent][args_count]",
-          "[process][parent][exit_code]",
-          "[process][parent][pgid]",
-          "[process][parent][ppid]",
-          "[process][parent][thread][id]",
-          "[process][parent][uptime]",
-          "[process][pgid]",
-          "[process][ppid]",
-          "[process][thread][id]",
-          "[process][uptime]",
-          "[server][as][number]",
-          "[server][nat][port]",
-          "[source][as][number]",
-          "[source][nat][port]",
-          "[tls][client][x509][public_key_exponent]",
-          "[tls][client][x509][public_key_size]",
-          "[tls][server][x509][public_key_exponent]",
-          "[tls][server][x509][public_key_size]",
-          "[url][port]"
-        ]
-        float_fields = [
-          "[event][risk_score]",
-          "[event][risk_score_norm]",
-          "[host][cpu][usage]",
-          "[vulnerability][score][base]",
-          "[vulnerability][score][environmental]",
-          "[vulnerability][score][temporal]"
-          ]
-
-        float_fields.each do |k|
-          field_value = event.get(k)
-          if field_value != nil
-            begin
-              converted_v = Float(field_value)
-              event.set(k, converted_v)
-            rescue
-              event.remove(k)
-            end
-          end
-        end
-
-        long_fields.each do |k|
-          field_value = event.get(k)
-          if field_value != nil
-            begin
-              converted_v = Integer(field_value)
-              event.set(k, converted_v)
-            rescue
-              event.remove(k)
-            end
-          end
-        end
-      '
-      tag_on_exception => "enrichment_data_type_ruby_exception"
-    }
-    ### Validate IP data types
-    mutate {
-      strip => ["[client][ip]", "[client][nat][ip]", "[destination][ip]", "[destination][nat][ip]", "[dns][resolved_ip]", "[host][ip]", "[network][forwarded_ip]", "[observer][ip]", "[server][ip]", "[server][nat][ip]", "[source][ip]", "[source][nat][ip]"]
-    }
-
-    # [log][source][ip]
-    if [log][source][ip] {
-      if [log][source][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![log][source][ip][0] {
-        if ![log][source][hostname] {
-          mutate {
-            copy => { "[log][source][ip]" => "[log][source][hostname]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [log][source][ip]" ]
-          remove_field => ["[log][source][ip]"]
-        }
-      }
-    }
-    # [source][ip]	[source][address]
-    if [source][ip] {
-      if [source][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![source][ip][0] {
-        if ![source][address] {
-          mutate {
-            copy => { "[source][ip]" => "[source][address]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [source][ip]" ]
-          remove_field => ["[source][ip]"]
-        }
-      }
-    }
-    # [client][ip]	[client][address]
-    if [client][ip] {
-      if [client][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![client][ip][0] {
-        if ![client][address] {
-          mutate {
-            copy => { "[client][ip]" => "[client][address]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [client][ip]" ]
-          remove_field => ["[client][ip]"]
-        }
-      }
-    }
-    # [client][nat][ip]	drop field
-    if [client][nat][ip] {
-      if [client][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![client][nat][ip][0] {
-        mutate {
-          remove_field => ["[client][nat][ip]"]
-          add_tag => [ "invalid ip format [client][nat][ip]" ]
-        }
-      }
-    }
-    # [destination][ip]	[destination][address]
-    if [destination][ip] {
-      if [destination][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![destination][ip][0] {
-        if ![destination][address] {
-          mutate {
-            copy => { "[destination][ip]" => "[destination][address]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [destination][ip]" ]
-          remove_field => ["[destination][ip]"]
-        }
-      }
-    }
-    # [destination][nat][ip] drop field
-    if [destination][nat][ip] {
-      if [destination][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![destination][nat][ip][0] {
-        mutate {
-          remove_field => ["[destination][nat][ip]"]
-          add_tag => [ "invalid ip format [destination][nat][ip]" ]
-        }
-      }
-    }
-    # [dns][resolved_ip]	[dns][answers][name]
-    if [dns][resolved_ip] {
-      if [dns][resolved_ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![dns][resolved_ip][0]{
-        if ![dns][answers][name] {
-          mutate {
-            copy => { "[dns][resolved_ip]" => "[dns][answers][name]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [dns][resolved_ip]" ]
-          remove_field => ["[dns][resolved_ip]"]
-        }
-      }
-    }
-    # [host][ip]	[host][hostname]
-    if [host][ip] {
-      if [host][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![host][ip][0] {
-        if ![host][hostname] {
-          mutate {
-            copy => { "[host][ip]" => "[host][hostname]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [host][ip]" ]
-          remove_field => ["[host][ip]"]
-        }
-      }
-    }
-    # network.forwarded_ip	drop field
-    if [network][forwarded_ip] {
-      if [network][forwarded_ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![network][forwarded_ip][0] {
-        mutate {
-          remove_field => ["[network][forwarded_ip]"]
-          add_tag => [ "invalid ip format [network][forwarded_ip]" ]
-        }
-      }
-    }
-    # [observer][ip]	[observer][name]
-    if [observer][ip] {
-      if [observer][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![observer][ip][0] {
-        if ![observer][name] {
-          mutate {
-            copy => { "[observer][ip]" => "[observer][name]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [observer][ip]" ]
-          remove_field => ["[observer][ip]"]
-        }
-      }
-    }
-    # related.ip	n/a if all fields are fixed before adding to related field then will not have issue
-
-    # [server][ip]	[server][address]
-    if [server][ip] {
-      if [server][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][ip][0] {
-        if ![server][name] {
-          mutate {
-            copy => { "[server][ip]" => "[server][name]" }
-          }
-        }
-        mutate {
-          add_tag => [ "invalid ip format [server][ip]" ]
-          remove_field => ["[server][ip]"]
-        }
-      }
-    }
-    # [server][nat][ip]	drop field
-    if [server][nat][ip] {
-      if [server][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][nat][ip][0] {
-        mutate {
-          remove_field => ["[server][nat][ip]"]
-          add_tag => [ "invalid ip format [server][nat][ip]" ]
-        }
-      }
-    }
-    # [source][nat][ip]	drop field
-    if [source][nat][ip] {
-      if [source][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][nat][ip][0] {
-        mutate {
-          remove_field => ["[source][nat][ip]"]
-          add_tag => [ "invalid ip format [source][nat][ip]" ]
-        }
-      }
-    }
-
-    ### Drop unparsed date fields and add tag
-    ruby {
-      code => '
-        # field names to check with their corresponding failure tags
-        date_fields_hash = {
-          "[event][created]" => "_dateparsefailure_ec",
-          "[event][end]" => "_dateparsefailure_ee",
-          "[event][ingested]" => "_dateparsefailure_ei",
-          "[event][start]" => "_dateparsefailure_es",
-          "[file][accessed]" => "_dateparsefailure_fa",
-          "[file][created]" => "_dateparsefailure_fc",
-          "[file][ctime]" => "_dateparsefailure_fct",
-          "[file][mtime]" => "_dateparsefailure_fm",
-          "[package][installed]" => "_dateparsefailure_pi",
-          "[process][parent][start]" => "_dateparsefailure_pps",
-          "[process][start]" => "_dateparsefailure_ps",
-          "[tls][client][not_after]" => "_dateparsefailure_tcna",
-          "[tls][client][not_before]" => "_dateparsefailure_tcnb",
-          "[tls][server][not_after]" => "_dateparsefailure_tsna",
-          "[tls][server][not_before]" => "_dateparsefailure_tsnb"
-        }
-        # for each field in the above has, check if the value in the event is a string
-        # if yes, remove that field and add the corresponding tag,
-        date_fields_hash.each do |k,v|
-          if event.get(k).is_a? String
-            event.remove(k)
-            event.tag(v)
-          end
-        end
-      '
-    }
-  }
-}
diff --git a/config/enrichments/97_output.conf b/config/enrichments/96_output.conf
similarity index 100%
rename from config/enrichments/97_output.conf
rename to config/enrichments/96_output.conf