10-logstash-syslog.conf

input {

# Listen on TCP/UDP port 5514 for CPPM messages
 tcp {
   port => 5514
   type => syslog
 }

 udp {
   port => 5514
   type => syslog
 }

}

filter {

  # if message contains CPPM message
  if [type] == "syslog" and  [message] =~ "CPPM" {
    
    # Parse out SYSLOG stuff (facility, level, etc)
    syslog_pri{}

    # Parse out SYSLOG message into compoents
    grok {
      match => { "message" => "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program} %{POSINT:syslog_pid} %{NONNEGINT:num_of_msgs} %{NONNEGINT:msg_num} %{GREEDYDATA:syslog_message}" }
    }

    # Parse out key-value pairs (fields separated by commas, data separated from fields by "=")
    kv { 
      source => "syslog_message"
      field_split => ","
      prefix => "CPPM_"
      add_tag => "CPPM_msg_grokkd"
    }

    # Replace timestamp with timestamp from CPPM timestamp field
    date {
      match => [ "CPPM_timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS", "ISO8601"  ]
      add_tag => "CPPM_timestamp_grokkd"
    }

    # The "Endpoint_Profile" message uses "updated_at" field instead of "timestamp" 
    if [syslog_program] == "CPPM_to_ELK_Endpoint_Profile" {
      date {
        match => [ "CPPM_updated_at", "yyyy-MM-dd HH:mm:ss.SSSSSS", "ISO8601"  ]
        add_tag => "CPPM_updated_at_grokkd"
      }
    }

    # Don't need to store syslog_message and message, so delete syslog_message field
    mutate {
	remove_field => [ "syslog_message" ]
    }

  } 
  # Process normal syslog message
  else if [type] == "syslog" {

    # Parse out SYSLOG stuff (facility, level, etc)
    syslog_pri{}

    # Parse SYSLOG message into compoents
    grok {
      match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_tag => "reg_syslog_grokkd"
    }
  }
}

output {

  # if parsing fails, write out messages to file
  if [type] == "syslog" and "_grokparsefailure" in [tags] {
    file { path => "/tmp/failed_syslog_events-%{+YYYY-MM-dd}" }
  } 
  # Send rest to elasticsearch cluster
  else {
    elasticsearch { host => "localhost"  cluster => "wlandata" }
  }

}