From a622562e76f98b208704f5daa5f73c61853ab442 Mon Sep 17 00:00:00 2001 From: JuanSheba Date: Tue, 20 Aug 2024 13:58:34 +0100 Subject: [PATCH 1/8] Refactor filter to set default 'direction' as 'upstream' and determine 'direction' dynamically based on IP match within homenets --- .../templates/default/sflow_tagging.conf.erb | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/resources/templates/default/sflow_tagging.conf.erb b/resources/templates/default/sflow_tagging.conf.erb index 214faff..9634567 100644 --- a/resources/templates/default/sflow_tagging.conf.erb +++ b/resources/templates/default/sflow_tagging.conf.erb @@ -1,32 +1,35 @@ filter { -<% @flow_nodes.each do |flow_node| %> - <% if !flow_node[:ipaddress].nil? and !flow_node["redborder"].nil? and flow_node["redborder"]["blocked"]!=true %> - if [tag] == 0 and [peer_ip_src] == "<%=flow_node[:ipaddress]%>" { - ruby { - code => " # loop in all the homenets - require 'ipaddr' - - internal = [] - <% flow_node["redborder"]["homenets"].each do |x| %> - internal.push(IPAddr.new('<%=x["value"]%>')) - <% end %> - - ip_src = IPAddr.new(event.get('ip_src')) - ip_dst = IPAddr.new(event.get('ip_dst')) - tag = 1 - if internal.any? {|subnet| subnet.include?(ip_src) } - if internal.any? {|subnet| subnet.include?(ip_dst) } - tag = 3 - else - tag = 2 - end - end + + # Default direction + mutate { + add_field => { + "direction" => "upstream" + } + } - event.set('tag', tag); + if ![tag] or [tag] == 0 { + <% @flow_nodes.select{|s| s[:ipaddress] and s["redborder"] and s["redborder"]["homenets"] and !s["redborder"]["blocked"]}.each do |flow_node| %> + if [peer_ip_src] == "<%=flow_node[:ipaddress]%>" { + # Determine if direction is different than "upstream" + ruby { + code => " require 'ipaddr' + + homenets = [<%=flow_node["redborder"]["homenets"].map{|h| "IPAddr.new('#{h["value"]}')"} .join(",")%>] - " + if homenets.any? {|subnet| subnet.include?(event.get('ip_src')) } + if homenets.any? {|subnet| subnet.include?(event.get('ip_dst')) } + event.set('direction', 'internal') + else + event.set('direction', 'downstream') + end + end + " } } - <% end unless flow_node["redborder"]["homenets"].nil? %> -<% end %> + <% end %> + } else if [tag] == 1 { + mutate { add_field => { "direction" => "downstream" } } + } else if [tag] == 3 { + mutate { add_field => { "direction" => "internal" } } + } } \ No newline at end of file From a233ae85f84664fbeec22498f5069811f4b9c689 Mon Sep 17 00:00:00 2001 From: JuanSheba Date: Tue, 20 Aug 2024 15:26:12 +0100 Subject: [PATCH 2/8] Refactor Logstash filter to simplify direction-based field renaming, set default values, handle observation_id, and optimize data processing --- .../default/sflow_normalization.conf.erb | 100 ++++++------------ 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/resources/templates/default/sflow_normalization.conf.erb b/resources/templates/default/sflow_normalization.conf.erb index ea9d528..4fa274d 100644 --- a/resources/templates/default/sflow_normalization.conf.erb +++ b/resources/templates/default/sflow_normalization.conf.erb @@ -16,39 +16,16 @@ filter { } } + # Set ip_proto if [ip_proto] == "udp" { - mutate { - add_field => { - "l4_proto" => 17 - } - } + mutate { add_field => { "l4_proto" => 17 } } } else if [ip_proto] == "tcp" { - mutate { - add_field => { - "l4_proto" => 6 - } - } + mutate { add_field => { "l4_proto" => 6 } } } + - # Egress - if [tag] == 2 { - mutate { - rename => { - "ip_src" => "lan_ip" - "ip_dst" => "wan_ip" - "port_src" => "lan_l4_port" - "port_dst" => "wan_l4_port" - "country_ip_src" => "lan_ip_country_code" - "country_ip_dst" => "wan_ip_country_code" - } - - add_field => { - "direction" => "upstream" - } - } - # Ingress - } else if [tag] == 1 { + if [direction] == "downstream" { # Ingress when direction is downstream mutate { rename => { "ip_src" => "wan_ip" @@ -57,59 +34,50 @@ filter { "port_dst" => "lan_l4_port" "country_ip_src" => "wan_ip_country_code" "country_ip_dst" => "lan_ip_country_code" - } - - add_field => { - "direction" => "downstream" + "mac_dst" => "client_mac" + "cisco_src_vlan" => "wan_vlan" + "cisco_dst_vlan" => "lan_vlan" + "src_vlan" => "wan_vlan" + "dst_vlan" => "lan_vlan" + "vlan_in" => "wan_vlan" + "vlan_out" => "lan_vlan" } } - } else if [tag] == 3 { + } else { # Egress when direction is upstream or internal.. mutate { rename => { - "ip_src" => "wan_ip" - "ip_dst" => "lan_ip" - "port_src" => "wan_l4_port" - "port_dst" => "lan_l4_port" - "country_ip_src" => "wan_ip_country_code" - "country_ip_dst" => "lan_ip_country_code" - } - - add_field => { - "direction" => "internal" + "ip_src" => "lan_ip" + "ip_dst" => "wan_ip" + "port_src" => "lan_l4_port" + "port_dst" => "wan_l4_port" + "country_ip_src" => "lan_ip_country_code" + "country_ip_dst" => "wan_ip_country_code" + "mac_src" => "client_mac" + "cisco_src_vlan" => "lan_vlan" + "cisco_dst_vlan" => "wan_vlan" + "src_vlan" => "lan_vlan" + "dst_vlan" => "wan_vlan" + "vlan_in" => "lan_vlan" + "vlan_out" => "wan_vlan" } } } - ruby { code => "event.set('timestamp', event.get('@timestamp').to_i); - event.set('bytes', event.get('bytes').to_i * (Integer(event.get('sampling_rate')) rescue 1)) - event.set('application_id_name', event.get('class').split('/').last) if event.get('class') - " - } - # Set observation_id: (if 4294967295 -> "default") if [tag2] and [tag2] != 4294967295 { mutate { replace => { "observation_id" => "%{tag2}" } } } - mutate { - - add_field => { - "type" => "sflowv5" - "ip_protocol_version" => 4 - "input_vrf" => 0 - "output_vrf" => 0 - } - - rename => { - "packets" => "pkts" - "export_proto_seqno" => "flow_sequence" - "peer_ip_src" => "sensor_ip" - } + # Set timestamp, bytes and application_id_name + ruby { code => " event.set('timestamp', event.get('@timestamp').to_i); + event.set('bytes', event.get('bytes').to_i * (Integer(event.get('sampling_rate')) rescue 1)) + event.set('application_id_name', event.get('class').split('/').last) if event.get('class') + " + } + mutate { remove_field => [ "ip_proto", "tag", "tag2", "stamp_updated", "event_type", "@version", "stamp_inserted", "writer_id", "timestamp_arrival", "@timestamp", "sampling_rate" ] - } -} - +} \ No newline at end of file From 48467fe27965e6b8e9a8dd17445aa84a56065b96 Mon Sep 17 00:00:00 2001 From: JuanSheba Date: Tue, 20 Aug 2024 15:27:01 +0100 Subject: [PATCH 3/8] Remove sflow_rename.conf template and corresponding resource from config.rb. --- resources/providers/config.rb | 9 --------- resources/templates/default/sflow_rename.conf.erb | 10 ---------- 2 files changed, 19 deletions(-) delete mode 100644 resources/templates/default/sflow_rename.conf.erb diff --git a/resources/providers/config.rb b/resources/providers/config.rb index edd280c..e22f916 100644 --- a/resources/providers/config.rb +++ b/resources/providers/config.rb @@ -257,15 +257,6 @@ notifies :restart, 'service[logstash]', :delayed end - template "#{pipelines_dir}/sflow/91_rename.conf" do - source 'sflow_rename.conf.erb' - owner user - group user - mode '0644' - ignore_failure true - cookbook 'logstash' - notifies :restart, 'service[logstash]', :delayed - end template "#{pipelines_dir}/sflow/99_output.conf" do source 'output_kafka.conf.erb' diff --git a/resources/templates/default/sflow_rename.conf.erb b/resources/templates/default/sflow_rename.conf.erb deleted file mode 100644 index 84e2146..0000000 --- a/resources/templates/default/sflow_rename.conf.erb +++ /dev/null @@ -1,10 +0,0 @@ -filter { - mutate { - rename => { - "cisco_src_vlan" => "lan_vlan" - "src_vlan" => "lan_vlan" - "cisco_dst_vlan" => "wan_vlan" - "dst_vlan" => "wan_vlan" - } - } -} From ada6b9791a59a8d93985cf213ffbd6742c78572f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20P=C3=A9rez?= Date: Tue, 20 Aug 2024 15:51:26 +0100 Subject: [PATCH 4/8] Fix --- resources/providers/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/providers/config.rb b/resources/providers/config.rb index edd280c..00513ca 100644 --- a/resources/providers/config.rb +++ b/resources/providers/config.rb @@ -657,7 +657,7 @@ mode '0644' ignore_failure true cookbook 'logstash' - variables(output_topic: '"rb_location') + variables(output_topic: 'rb_location') notifies :restart, 'service[logstash]', :delayed end end From b4df9a6c039c2f4930832e5525da17f6e9c2900d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20P=C3=A9rez?= Date: Tue, 20 Aug 2024 16:07:04 +0100 Subject: [PATCH 5/8] Release 2.3.4 --- CHANGELOG.md | 6 ++++++ resources/metadata.rb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9c7c3..6ba9970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ cookbook-logstash CHANGELOG =============== +## 2.3.4 + + - Pablo Pérez + - [e5d879a] Merge pull request #56 from redBorder/bugfix/#18398_fix_radius_output + - [ada6b97] Fix + ## 2.3.3 - Miguel Negron diff --git a/resources/metadata.rb b/resources/metadata.rb index 7985f08..51730a5 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,4 +3,4 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures cookbook-logstash' -version '2.3.3' +version '2.3.4' From ceb7e0b96e8cdafbec2752a075577fea04816b06 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Thu, 22 Aug 2024 11:28:17 +0100 Subject: [PATCH 6/8] auto lint --- resources/providers/config.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/providers/config.rb b/resources/providers/config.rb index e22f916..ec2f127 100644 --- a/resources/providers/config.rb +++ b/resources/providers/config.rb @@ -257,7 +257,6 @@ notifies :restart, 'service[logstash]', :delayed end - template "#{pipelines_dir}/sflow/99_output.conf" do source 'output_kafka.conf.erb' owner user From be75f190a3d5f664bf39caae68ea3e111c01eed2 Mon Sep 17 00:00:00 2001 From: Luis Blanco Date: Thu, 22 Aug 2024 17:10:19 +0100 Subject: [PATCH 7/8] auto bump --- CHANGELOG.md | 7 +++++++ resources/metadata.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba9970..389d050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ cookbook-logstash CHANGELOG =============== +## 2.4.0 + + - JuanSheba + - [48467fe] Remove sflow_rename.conf template and corresponding resource from config.rb. + - [a233ae8] Refactor Logstash filter to simplify direction-based field renaming, set default values, handle observation_id, and optimize data processing + - [a622562] Refactor filter to set default 'direction' as 'upstream' and determine 'direction' dynamically based on IP match within homenets + ## 2.3.4 - Pablo Pérez diff --git a/resources/metadata.rb b/resources/metadata.rb index 51730a5..78c9010 100644 --- a/resources/metadata.rb +++ b/resources/metadata.rb @@ -3,4 +3,4 @@ maintainer_email 'git@redborder.com' license 'AGPL-3.0' description 'Installs/Configures cookbook-logstash' -version '2.3.4' +version '2.4.0' From c9b2ba4e5ee39851204408b1e5957a53282b838a Mon Sep 17 00:00:00 2001 From: Luis Blanco <108473576+ljblancoredborder@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:36:31 +0100 Subject: [PATCH 8/8] Update CHANGELOG.md --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 389d050..2893bd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,8 @@ cookbook-logstash CHANGELOG - [48467fe] Remove sflow_rename.conf template and corresponding resource from config.rb. - [a233ae8] Refactor Logstash filter to simplify direction-based field renaming, set default values, handle observation_id, and optimize data processing - [a622562] Refactor filter to set default 'direction' as 'upstream' and determine 'direction' dynamically based on IP match within homenets - -## 2.3.4 - - Pablo Pérez - - [e5d879a] Merge pull request #56 from redBorder/bugfix/#18398_fix_radius_output - - [ada6b97] Fix + - [ada6b97] Fix Radius output ## 2.3.3