Merge pull request #46 from logicmonitor/DEV-93860-support-device-les…

…s-logs

DEV-93860 added config to support deviceless logs and exclude metadata.

README.md

@@ -69,4 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
 | `debug` | When `true`, logs more information to the fluentd console. |
 | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. |
+| `include_metadata` | When `true`, appends additional metadata to the log. default `false`.  |
+| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` and `namespace` when `true`. default `false`. |
+| `metadata_to_exclude` | `String array` of keys of metadata to exclude. default `empty` |
+
 

fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '1.0.1'
+  spec.version                        = '1.0.2'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

lib/fluent/plugin/out_lm.rb

@@ -36,7 +36,11 @@ class LmOutput < BufferedOutput
     config_param :log_source,  :string, :default => "lm-logs-fluentd"
 
     config_param :version_id,  :string, :default => "version_id"
-
+
+    config_param :device_less_logs,  :bool, :default => false
+
+    config_param :metadata_to_exclude,  :array, default: [], value_type: :string
+
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
     # If the configuration is invalid, raise Fluent::ConfigError.
@@ -73,27 +77,30 @@ def write(chunk)
       events = []
       chunk.msgpack_each do |(tag, time, record)|
         event = process_record(tag,time,record)
-        events.push(event)
+        if event != nil
+          events.push(event)
+        end
       end
       send_batch(events)
     end
 
     def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
-
-      if record["_lm.resourceId"] == nil
-          @resource_mapping.each do |key, value|
-            k = value
-            nestedVal = record
-            key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
-            if nestedVal != nil
-              resource_map[k] = nestedVal
+      if !@device_less_logs
+        if record["_lm.resourceId"] == nil
+            @resource_mapping.each do |key, value|
+              k = value
+              nestedVal = record
+              key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
+              if nestedVal != nil
+                resource_map[k] = nestedVal
+              end
             end
-          end
-        lm_event["_lm.resourceId"] = resource_map
-      else
-        lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+          lm_event["_lm.resourceId"] = resource_map
+        else
+          lm_event["_lm.resourceId"] = record["_lm.resourceId"]
+        end
       end
 
       if record["timestamp"] != nil
@@ -104,22 +111,32 @@ def process_record(tag, time, record)
 
       if @include_metadata
         record.each do |key, value|
-          if key != "timestamp" || key != "_lm.resourceId"
+          case key
+          when *@metadata_to_exclude.concat(["timestamp","_lm.resourceId","log","message"])
+          else    
               lm_event["#{key}"] = value
 
               if @force_encoding != ""
                   lm_event["#{key}"] = lm_event["#{key}"].force_encoding(@force_encoding).encode("UTF-8")
               end
           end
         end
-      else
-        lm_event["message"] = record["message"]
-
-        if @force_encoding != ""
-          lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
-        end
+      end
+      lm_event["message"] = record["message"]
+
+      if @force_encoding != ""
+        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
       end
 
+      if @device_less_logs
+        if record["service"]==nil || record["namespace"] == nil
+          log.error "When device_less_logs is set \'true\', record must have \'service\' and \'namespace\'. Ignoring this event #{lm_event}."
+          return nil
+        else
+          lm_event["service"] = record["service"]
+          lm_event["namespace"] = record["namespace"] 
+        end
+      end
       return lm_event
     end
 

test/plugin/test_device_less_logs.rb

@@ -0,0 +1,101 @@
+require "fluent/test"
+require "fluent/test/helpers"
+require "fluent/test/driver/output"
+require "./lib/fluent/plugin/out_lm"
+
+class FluentLMTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::LmOutput).configure(conf)
+  end
+
+  def create_valid_subject
+    create_driver(%[
+        api_key = foo
+      ]).instance
+  end
+  sub_test_case "device_less_logs" do
+    test "if device_less_logs is true, resource_mapping to be skipped" do
+      plugin = create_driver(%[
+        resource_mapping {"someProp": "lm_property"} 
+        device_less_logs true
+      ]).instance
+      tag = "lm.test"
+      time = Time.parse("2020-08-23T00:53:15+00:00").to_i
+      record = {"message" => "Hello from test",  
+                "timestamp" => "2020-10-30T00:29:08.629701504Z", 
+                "tag" => "lm.test", 
+                "meta1" => "metadata1", 
+                "meta2" => "metadata2", 
+                "service" => "lm-service", 
+                "namespace" => "lm-namepsace",
+                "someProp" => "someVal"}
+
+      result = plugin.process_record(tag, time, record)
+
+      expected = {
+          "message" => "Hello from test",
+          "timestamp" => "2020-10-30T00:29:08.629701504Z",
+          "service" => "lm-service", 
+          "namespace" => "lm-namepsace"
+      }
+      assert_equal expected, result
+    end
+
+    test "if device_less_logs and include_metadata true, skip metadata defined  in metadata_to_exclude and resource_mapping" do
+      plugin = create_driver(%[
+        resource_mapping {"a.b": "lm_property"} 
+        device_less_logs true
+        include_metadata true
+        metadata_to_exclude ["meta1", "meta3"]
+      ]).instance
+      tag = "lm.test"
+      time = Time.parse("2020-08-23T00:53:15+00:00").to_i
+      record = {
+                "message" => "Hello from test",  
+                "timestamp" => "2020-10-30T00:29:08.629701504Z" , 
+                "meta1" => "testMeta1" , 
+                "meta2" => "testMeta2", 
+                "meta3" => "testMeta3", 
+                "meta4" => "testMeta4", 
+                "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, 
+                "service" => "lm-service", 
+                "namespace" => "lm-namepsace" }
+
+      result = plugin.process_record(tag, time, record)
+
+      expected = {
+          "message" => "Hello from test",
+          "timestamp" => "2020-10-30T00:29:08.629701504Z",
+          "meta2" => "testMeta2",
+          "meta4" => "testMeta4", 
+          "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}},
+          "service" => "lm-service",
+          "namespace" => "lm-namepsace"
+      }
+      assert_equal expected, result
+    end
+
+    test "when device_less_logs is true record must have \'service\' and \'namespace\'" do
+      plugin = create_driver(%[
+        resource_mapping {"a.b": "lm_property"} 
+        device_less_logs true
+      ]).instance
+      tag = "lm.test"
+      time = Time.parse("2020-08-23T00:53:15+00:00").to_i
+      record = {"message" => "Hello from test",  
+                "timestamp" => "2020-10-30T00:29:08.629701504Z", 
+                "service" => "lm-service" }
+
+      result = plugin.process_record(tag, time, record)
+
+      expected = nil
+      assert_equal expected, result
+    end
+  end
+end

test/plugin/test_out_lm.rb

@@ -147,5 +147,35 @@ def create_valid_subject
 
       assert_equal expected, result
     end
+
+    test "include_metadata true, try to include everything except in metadata_to_exclude" do
+      plugin = create_driver(%[
+        resource_mapping {"someProp": "lm_property"} 
+        include_metadata true
+        metadata_to_exclude ["meta1", "meta3","someProp"]
+      ]).instance
+      tag = "lm.test"
+      time = Time.parse("2020-08-23T00:53:15+00:00").to_i
+      record = {"message" => "Hello from test",  
+                "timestamp" => "2020-10-30T00:29:08.629701504Z" , 
+                "meta1" => "testMeta1" , 
+                "meta2" => "testMeta2", 
+                "meta3" => "testMeta3", 
+                "meta4" => "testMeta4", 
+                "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, 
+                "someProp" => "resourcePropVal" }
+
+      result = plugin.process_record(tag, time, record)
+
+      expected = {
+          "message" => "Hello from test",
+          "timestamp" => "2020-10-30T00:29:08.629701504Z",
+          "_lm.resourceId" => {"lm_property" => "resourcePropVal"},
+          "meta2" => "testMeta2",
+          "meta4" => "testMeta4", 
+          "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}},
+      }
+      assert_equal expected, result
+    end
   end
 end