Skip to content

Commit

Permalink
Merge pull request #57 from redBorder/development
Browse files Browse the repository at this point in the history
Release 2.4.0
  • Loading branch information
jsotofernandez authored Aug 23, 2024
2 parents bdaefe0 + c9b2ba4 commit cf6df39
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 114 deletions.
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
cookbook-logstash CHANGELOG
===============

## 2.4.0

- JuanSheba
- [48467fe] Remove sflow_rename.conf template and corresponding resource from config.rb.
- [a233ae8] Refactor Logstash filter to simplify direction-based field renaming, set default values, handle observation_id, and optimize data processing
- [a622562] Refactor filter to set default 'direction' as 'upstream' and determine 'direction' dynamically based on IP match within homenets
- Pablo Pérez
- [ada6b97] Fix Radius output

## 2.3.3

- Miguel Negron
Expand Down
2 changes: 1 addition & 1 deletion resources/metadata.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
maintainer_email 'git@redborder.com'
license 'AGPL-3.0'
description 'Installs/Configures cookbook-logstash'
version '2.3.3'
version '2.4.0'
12 changes: 1 addition & 11 deletions resources/providers/config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -257,16 +257,6 @@
notifies :restart, 'service[logstash]', :delayed
end

template "#{pipelines_dir}/sflow/91_rename.conf" do
source 'sflow_rename.conf.erb'
owner user
group user
mode '0644'
ignore_failure true
cookbook 'logstash'
notifies :restart, 'service[logstash]', :delayed
end

template "#{pipelines_dir}/sflow/99_output.conf" do
source 'output_kafka.conf.erb'
owner user
Expand Down Expand Up @@ -657,7 +647,7 @@
mode '0644'
ignore_failure true
cookbook 'logstash'
variables(output_topic: '"rb_location')
variables(output_topic: 'rb_location')
notifies :restart, 'service[logstash]', :delayed
end
end
Expand Down
100 changes: 34 additions & 66 deletions resources/templates/default/sflow_normalization.conf.erb
Original file line number Diff line number Diff line change
Expand Up @@ -16,39 +16,16 @@ filter {
}
}


# Set ip_proto
if [ip_proto] == "udp" {
mutate {
add_field => {
"l4_proto" => 17
}
}
mutate { add_field => { "l4_proto" => 17 } }
} else if [ip_proto] == "tcp" {
mutate {
add_field => {
"l4_proto" => 6
}
}
mutate { add_field => { "l4_proto" => 6 } }
}


# Egress
if [tag] == 2 {
mutate {
rename => {
"ip_src" => "lan_ip"
"ip_dst" => "wan_ip"
"port_src" => "lan_l4_port"
"port_dst" => "wan_l4_port"
"country_ip_src" => "lan_ip_country_code"
"country_ip_dst" => "wan_ip_country_code"
}

add_field => {
"direction" => "upstream"
}
}
# Ingress
} else if [tag] == 1 {
if [direction] == "downstream" { # Ingress when direction is downstream
mutate {
rename => {
"ip_src" => "wan_ip"
Expand All @@ -57,59 +34,50 @@ filter {
"port_dst" => "lan_l4_port"
"country_ip_src" => "wan_ip_country_code"
"country_ip_dst" => "lan_ip_country_code"
}

add_field => {
"direction" => "downstream"
"mac_dst" => "client_mac"
"cisco_src_vlan" => "wan_vlan"
"cisco_dst_vlan" => "lan_vlan"
"src_vlan" => "wan_vlan"
"dst_vlan" => "lan_vlan"
"vlan_in" => "wan_vlan"
"vlan_out" => "lan_vlan"
}
}
} else if [tag] == 3 {
} else { # Egress when direction is upstream or internal..
mutate {
rename => {
"ip_src" => "wan_ip"
"ip_dst" => "lan_ip"
"port_src" => "wan_l4_port"
"port_dst" => "lan_l4_port"
"country_ip_src" => "wan_ip_country_code"
"country_ip_dst" => "lan_ip_country_code"
}

add_field => {
"direction" => "internal"
"ip_src" => "lan_ip"
"ip_dst" => "wan_ip"
"port_src" => "lan_l4_port"
"port_dst" => "wan_l4_port"
"country_ip_src" => "lan_ip_country_code"
"country_ip_dst" => "wan_ip_country_code"
"mac_src" => "client_mac"
"cisco_src_vlan" => "lan_vlan"
"cisco_dst_vlan" => "wan_vlan"
"src_vlan" => "lan_vlan"
"dst_vlan" => "wan_vlan"
"vlan_in" => "lan_vlan"
"vlan_out" => "wan_vlan"
}
}
}

ruby { code => "event.set('timestamp', event.get('@timestamp').to_i);
event.set('bytes', event.get('bytes').to_i * (Integer(event.get('sampling_rate')) rescue 1))
event.set('application_id_name', event.get('class').split('/').last) if event.get('class')
"
}

# Set observation_id: (if 4294967295 -> "default")
if [tag2] and [tag2] != 4294967295 {
mutate { replace => { "observation_id" => "%{tag2}" } }
}

mutate {

add_field => {
"type" => "sflowv5"
"ip_protocol_version" => 4
"input_vrf" => 0
"output_vrf" => 0
}

rename => {
"packets" => "pkts"
"export_proto_seqno" => "flow_sequence"
"peer_ip_src" => "sensor_ip"
}
# Set timestamp, bytes and application_id_name
ruby { code => " event.set('timestamp', event.get('@timestamp').to_i);
event.set('bytes', event.get('bytes').to_i * (Integer(event.get('sampling_rate')) rescue 1))
event.set('application_id_name', event.get('class').split('/').last) if event.get('class')
"
}

mutate {
remove_field => [ "ip_proto", "tag", "tag2", "stamp_updated", "event_type", "@version", "stamp_inserted", "writer_id", "timestamp_arrival", "@timestamp", "sampling_rate" ]

}


}

}
10 changes: 0 additions & 10 deletions resources/templates/default/sflow_rename.conf.erb

This file was deleted.

55 changes: 29 additions & 26 deletions resources/templates/default/sflow_tagging.conf.erb
Original file line number Diff line number Diff line change
@@ -1,32 +1,35 @@
filter {
<% @flow_nodes.each do |flow_node| %>
<% if !flow_node[:ipaddress].nil? and !flow_node["redborder"].nil? and flow_node["redborder"]["blocked"]!=true %>
if [tag] == 0 and [peer_ip_src] == "<%=flow_node[:ipaddress]%>" {
ruby {
code => " # loop in all the homenets
require 'ipaddr'

internal = []
<% flow_node["redborder"]["homenets"].each do |x| %>
internal.push(IPAddr.new('<%=x["value"]%>'))
<% end %>

ip_src = IPAddr.new(event.get('ip_src'))
ip_dst = IPAddr.new(event.get('ip_dst'))
tag = 1
if internal.any? {|subnet| subnet.include?(ip_src) }
if internal.any? {|subnet| subnet.include?(ip_dst) }
tag = 3
else
tag = 2
end
end

# Default direction
mutate {
add_field => {
"direction" => "upstream"
}
}

event.set('tag', tag);
if ![tag] or [tag] == 0 {
<% @flow_nodes.select{|s| s[:ipaddress] and s["redborder"] and s["redborder"]["homenets"] and !s["redborder"]["blocked"]}.each do |flow_node| %>
if [peer_ip_src] == "<%=flow_node[:ipaddress]%>" {
# Determine if direction is different than "upstream"
ruby {
code => " require 'ipaddr'

homenets = [<%=flow_node["redborder"]["homenets"].map{|h| "IPAddr.new('#{h["value"]}')"} .join(",")%>]

"
if homenets.any? {|subnet| subnet.include?(event.get('ip_src')) }
if homenets.any? {|subnet| subnet.include?(event.get('ip_dst')) }
event.set('direction', 'internal')
else
event.set('direction', 'downstream')
end
end
"
}
}
<% end unless flow_node["redborder"]["homenets"].nil? %>
<% end %>
<% end %>
} else if [tag] == 1 {
mutate { add_field => { "direction" => "downstream" } }
} else if [tag] == 3 {
mutate { add_field => { "direction" => "internal" } }
}
}

0 comments on commit cf6df39

Please sign in to comment.