Merge branch 'master' of github.com:redBorder/cookbook-logstash into …

…bugfix/19477_wrong_sensor_name_in_sflow_pipeline

CHANGELOG.md

@@ -1,6 +1,34 @@
 cookbook-logstash CHANGELOG
 ===============
 
+## 6.1.0
+
+  - David Vanhoucke
+    - [51f427d] add vlan normalization
+
+## 6.0.0
+
+  - Miguel Negrón
+    - [94bcd8f] Merge pull request #83 from redBorder/bugfix/#19815_fix_splitting_traffic_sflow_pipeline
+    - [940b53d] Fix last row
+    - [6feffeb] Load interfaces proxy from the role instead of role
+  - nilsver
+    - [7cd2f65] Release
+  - Miguel Álvarez
+    - [fcbea79] Add new filters for flow and intrusion (#79)
+
+## 5.0.0
+
+  - Miguel Álvarez
+    - [fcbea79] Add new filters for flow and intrusion (#79)
+
+## 4.0.0
+
+  - manegron
+    - [bd193fc] remove space
+    - [69bb4b7] Dont incident_enrichment if is already enriched
+    - [43b5113] Remove alarms from vault pipeline
+
 ## 3.3.0
 
   - Miguel Negrón

resources/metadata.rb

@@ -3,4 +3,4 @@
 maintainer_email 'git@redborder.com'
 license          'AGPL-3.0'
 description      'Installs/Configures cookbook-logstash'
-version          '3.3.0'
+version          '6.1.0'

resources/providers/config.rb

@@ -189,14 +189,9 @@
         notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
       end
 
-      template "#{pipelines_dir}/vault/06_alarms.conf" do
-        source 'vault_alarms.conf.erb'
-        owner user
-        group user
-        mode '0644'
-        ignore_failure true
-        cookbook 'logstash'
-        notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
+      # We dont need this file anymore as is parsed by rsyslog
+      file "#{pipelines_dir}/vault/06_alarms.conf" do
+        action :delete
       end
 
       # Renamed so we clean the old file
@@ -368,6 +363,19 @@
         notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
       end
 
+      memcached_servers = node['redborder']['memcached']['hosts']
+
+      template "#{pipelines_dir}/netflow/05_threat_intelligence.conf" do
+        source 'netflow_threat_intelligence.conf.erb'
+        owner user
+        group user
+        mode '0644'
+        ignore_failure true
+        cookbook 'logstash'
+        variables(memcached_servers: memcached_servers)
+        notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
+      end
+
       template "#{pipelines_dir}/netflow/90_splitflow.conf" do
         source 'netflow_splitflow.conf.erb'
         owner user
@@ -917,6 +925,17 @@
         notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
       end
 
+      template "#{pipelines_dir}/intrusion/07_threat_intelligence.conf" do
+        source 'intrusion_threat_intelligence.conf.erb'
+        owner user
+        group user
+        mode '0644'
+        ignore_failure true
+        cookbook 'logstash'
+        variables(memcached_servers: memcached_servers)
+        notifies :restart, 'service[logstash]', :delayed unless node['redborder']['leader_configuring']
+      end
+
       template "#{pipelines_dir}/intrusion/98_encode.conf" do
         source 'intrusion_encode.conf.erb'
         owner user

resources/templates/default/intrusion_incident_enrichment.conf.erb

@@ -1,7 +1,9 @@
 filter {
-  incident_enrichment { 
-    incident_fields => ["src","src_port", "dst", "dst_port"]
-    source => "redBorder Intrusion"
-    incidents_priority_filter => "<%= @intrusion_incidents_priority_filter %>"
+  if ![incident_uuid] {
+    incident_enrichment {
+      incident_fields => ["src","src_port", "dst", "dst_port"]
+      source => "redBorder Intrusion"
+      incidents_priority_filter => "<%= @intrusion_incidents_priority_filter %>"
+    }
   }
 }

resources/templates/default/intrusion_threat_intelligence.conf.erb

@@ -0,0 +1,13 @@
+filter {
+  threatintelligence {
+    memcached_servers => <%=@memcached_servers%>
+    key_mapping => {
+      "src" => "src_is_malicious"
+      "dst" => "dst_is_malicious"
+      "public_ip" => "src_is_malicious"
+      "sha256" => "sha256_is_malicious"
+      "file_uri" => "file_uri_is_malicious"
+      "file_hostname" => "file_hostname_is_malicious"
+    }
+  }
+}

resources/templates/default/netflow_threat_intelligence.conf.erb

@@ -0,0 +1,10 @@
+filter {
+  threatintelligence {
+    memcached_servers => <%=@memcached_servers%>
+    key_mapping => {
+      "lan_ip" => "lan_ip_is_malicious"
+      "wan_ip" => "wan_ip_is_malicious"
+      "public_ip" => "public_ip_is_malicious"
+    }
+  }
+}

resources/templates/default/sflow_normalization.conf.erb

@@ -37,10 +37,28 @@ filter {
         "mac_dst" => "client_mac"
         "cisco_src_vlan" => "wan_vlan"
         "cisco_dst_vlan" => "lan_vlan"
-        "src_vlan" => "wan_vlan"
-        "dst_vlan" => "lan_vlan"
-        "vlan_in" => "wan_vlan"
-        "vlan_out" => "lan_vlan"
+        }
+    }
+    if ![tag] or [tag] == 0 {
+      mutate {
+        rename => {
+          "vlan" => "lan_vlan"
+          "src_vlan" => "lan_vlan"
+          "dst_vlan" => "wan_vlan"
+          "vlan_in" => "lan_vlan"
+          "vlan_out" => "wan_vlan"
+        }
+        }
+    }
+    else {
+      mutate {
+        rename => {
+          "vlan" => "wan_vlan"
+          "src_vlan" => "wan_vlan"
+          "dst_vlan" => "lan_vlan"
+          "vlan_in" => "wan_vlan"
+          "vlan_out" => "lan_vlan"
+        }
       }
     }
   } else { # Egress when direction is upstream or internal..
@@ -55,6 +73,7 @@ filter {
         "mac_src" => "client_mac"
         "cisco_src_vlan" => "lan_vlan"
         "cisco_dst_vlan" => "wan_vlan"
+        "vlan" => "lan_vlan"
         "src_vlan" => "lan_vlan"
         "dst_vlan" => "wan_vlan"
         "vlan_in" => "lan_vlan"

resources/templates/default/sflow_tagging.conf.erb

@@ -49,11 +49,14 @@ filter {
 
             tag2 = event.get('tag2')
             observation_tag2 = tag2
-
-            <% if proxy_node['interfaces'] && !proxy_node['interfaces'].empty? %>
-              <% proxy_node['interfaces'].each do |iface_key, _orig_iface| %>
-              <% unless proxy_node['interfaces'][iface_key]['observation_id_filters'].empty? %>
-              observation_id_filters = <%= proxy_node['interfaces'][iface_key]['observation_id_filters'].to_s.gsub('"', "'") %>
+
+            <% role_name = "rBsensor-#{proxy_node.name.gsub('rbproxy-','')}" %>
+            <% proxy_role = Chef::Role.load role_name rescue nil %>
+
+            <% if proxy_role && proxy_role.override_attributes && proxy_role.override_attributes['redborder'] && proxy_role.override_attributes['redborder']['interfaces'] && !proxy_role.override_attributes['redborder']['interfaces'].empty? %>
+              <% proxy_role.override_attributes['redborder']['interfaces'].each do |iface_key, _orig_iface| %>
+              <% unless proxy_role.override_attributes['redborder']['interfaces'][iface_key]['observation_id_filters'].empty? %>
+              observation_id_filters = <%= proxy_role.override_attributes['redborder']['interfaces'][iface_key]['observation_id_filters'].to_s.gsub('"', "'") %>
 
               observation_id_filters.each do |filter|
                 subnet = IPAddr.new(filter['subnet'])
@@ -89,4 +92,5 @@ filter {
     <% end %>
     }
   <% end %>
-}
+}
+

resources/templates/default/vault_incident_enrichment.conf.erb

@@ -1,13 +1,15 @@
 filter {
-  incident_enrichment {
-    incident_fields => ["fromhost_ip"]
-    source => "redBorder Vault"
-    incidents_priority_filter => "<%= @vault_incidents_priority_filter %>"
-    field_scores => {
-      "fromhost_ip" => 100
-    }
-    field_map => {
-      "fromhost_ip" => "ip"
+  if ![incident_uuid] {
+    incident_enrichment {
+      incident_fields => ["fromhost_ip"]
+      source => "redBorder Vault"
+      incidents_priority_filter => "<%= @vault_incidents_priority_filter %>"
+      field_scores => {
+        "fromhost_ip" => 100
+      }
+      field_map => {
+        "fromhost_ip" => "ip"
+      }
     }
   }
 }