diff --git a/README.md b/README.md index a71c1ec..127ba4a 100644 --- a/README.md +++ b/README.md @@ -69,4 +69,8 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm- | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. | | `debug` | When `true`, logs more information to the fluentd console. | | `force_encoding` | Specify charset when logs contains invalid utf-8 characters. | +| `include_metadata` | When `true`, appends additional metadata to the log. default `false`. | +| `device_less_logs` | When `true`, do not map log with any resource. record must have `service` and `namespace` when `true`. default `false`. | +| `metadata_to_exclude` | `String array` of keys of metadata to exclude. default `empty` | + diff --git a/fluent-plugin-lm-logs.gemspec b/fluent-plugin-lm-logs.gemspec index 7960656..d497031 100644 --- a/fluent-plugin-lm-logs.gemspec +++ b/fluent-plugin-lm-logs.gemspec @@ -5,7 +5,7 @@ Gem::Specification.new do |spec| spec.name = "fluent-plugin-lm-logs" - spec.version = '1.0.1' + spec.version = '1.0.2' spec.authors = ["LogicMonitor"] spec.email = "rubygems@logicmonitor.com" spec.summary = "LogicMonitor logs fluentd output plugin" diff --git a/lib/fluent/plugin/out_lm.rb b/lib/fluent/plugin/out_lm.rb index 2adbd02..9b385f4 100644 --- a/lib/fluent/plugin/out_lm.rb +++ b/lib/fluent/plugin/out_lm.rb @@ -36,7 +36,11 @@ class LmOutput < BufferedOutput config_param :log_source, :string, :default => "lm-logs-fluentd" config_param :version_id, :string, :default => "version_id" - + + config_param :device_less_logs, :bool, :default => false + + config_param :metadata_to_exclude, :array, default: [], value_type: :string + # This method is called before starting. # 'conf' is a Hash that includes configuration parameters. # If the configuration is invalid, raise Fluent::ConfigError. @@ -73,7 +77,9 @@ def write(chunk) events = [] chunk.msgpack_each do |(tag, time, record)| event = process_record(tag,time,record) - events.push(event) + if event != nil + events.push(event) + end end send_batch(events) end @@ -81,19 +87,20 @@ def write(chunk) def process_record(tag, time, record) resource_map = {} lm_event = {} - - if record["_lm.resourceId"] == nil - @resource_mapping.each do |key, value| - k = value - nestedVal = record - key.to_s.split('.').each { |x| nestedVal = nestedVal[x] } - if nestedVal != nil - resource_map[k] = nestedVal + if !@device_less_logs + if record["_lm.resourceId"] == nil + @resource_mapping.each do |key, value| + k = value + nestedVal = record + key.to_s.split('.').each { |x| nestedVal = nestedVal[x] } + if nestedVal != nil + resource_map[k] = nestedVal + end end - end - lm_event["_lm.resourceId"] = resource_map - else - lm_event["_lm.resourceId"] = record["_lm.resourceId"] + lm_event["_lm.resourceId"] = resource_map + else + lm_event["_lm.resourceId"] = record["_lm.resourceId"] + end end if record["timestamp"] != nil @@ -104,7 +111,9 @@ def process_record(tag, time, record) if @include_metadata record.each do |key, value| - if key != "timestamp" || key != "_lm.resourceId" + case key + when *@metadata_to_exclude.concat(["timestamp","_lm.resourceId","log","message"]) + else lm_event["#{key}"] = value if @force_encoding != "" @@ -112,14 +121,22 @@ def process_record(tag, time, record) end end end - else - lm_event["message"] = record["message"] - - if @force_encoding != "" - lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8") - end + end + lm_event["message"] = record["message"] + + if @force_encoding != "" + lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8") end + if @device_less_logs + if record["service"]==nil || record["namespace"] == nil + log.error "When device_less_logs is set \'true\', record must have \'service\' and \'namespace\'. Ignoring this event #{lm_event}." + return nil + else + lm_event["service"] = record["service"] + lm_event["namespace"] = record["namespace"] + end + end return lm_event end diff --git a/test/plugin/test_device_less_logs.rb b/test/plugin/test_device_less_logs.rb new file mode 100644 index 0000000..d32acbe --- /dev/null +++ b/test/plugin/test_device_less_logs.rb @@ -0,0 +1,101 @@ +require "fluent/test" +require "fluent/test/helpers" +require "fluent/test/driver/output" +require "./lib/fluent/plugin/out_lm" + +class FluentLMTest < Test::Unit::TestCase + include Fluent::Test::Helpers + + def setup + Fluent::Test.setup + end + + def create_driver(conf = CONFIG) + Fluent::Test::Driver::Output.new(Fluent::LmOutput).configure(conf) + end + + def create_valid_subject + create_driver(%[ + api_key = foo + ]).instance + end + sub_test_case "device_less_logs" do + test "if device_less_logs is true, resource_mapping to be skipped" do + plugin = create_driver(%[ + resource_mapping {"someProp": "lm_property"} + device_less_logs true + ]).instance + tag = "lm.test" + time = Time.parse("2020-08-23T00:53:15+00:00").to_i + record = {"message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z", + "tag" => "lm.test", + "meta1" => "metadata1", + "meta2" => "metadata2", + "service" => "lm-service", + "namespace" => "lm-namepsace", + "someProp" => "someVal"} + + result = plugin.process_record(tag, time, record) + + expected = { + "message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z", + "service" => "lm-service", + "namespace" => "lm-namepsace" + } + assert_equal expected, result + end + + test "if device_less_logs and include_metadata true, skip metadata defined in metadata_to_exclude and resource_mapping" do + plugin = create_driver(%[ + resource_mapping {"a.b": "lm_property"} + device_less_logs true + include_metadata true + metadata_to_exclude ["meta1", "meta3"] + ]).instance + tag = "lm.test" + time = Time.parse("2020-08-23T00:53:15+00:00").to_i + record = { + "message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z" , + "meta1" => "testMeta1" , + "meta2" => "testMeta2", + "meta3" => "testMeta3", + "meta4" => "testMeta4", + "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, + "service" => "lm-service", + "namespace" => "lm-namepsace" } + + result = plugin.process_record(tag, time, record) + + expected = { + "message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z", + "meta2" => "testMeta2", + "meta4" => "testMeta4", + "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, + "service" => "lm-service", + "namespace" => "lm-namepsace" + } + assert_equal expected, result + end + + test "when device_less_logs is true record must have \'service\' and \'namespace\'" do + plugin = create_driver(%[ + resource_mapping {"a.b": "lm_property"} + device_less_logs true + ]).instance + tag = "lm.test" + time = Time.parse("2020-08-23T00:53:15+00:00").to_i + record = {"message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z", + "service" => "lm-service" } + + result = plugin.process_record(tag, time, record) + + expected = nil + assert_equal expected, result + end + end +end \ No newline at end of file diff --git a/test/plugin/test_out_lm.rb b/test/plugin/test_out_lm.rb index 01fbe0f..9cf0b28 100644 --- a/test/plugin/test_out_lm.rb +++ b/test/plugin/test_out_lm.rb @@ -147,5 +147,35 @@ def create_valid_subject assert_equal expected, result end + + test "include_metadata true, try to include everything except in metadata_to_exclude" do + plugin = create_driver(%[ + resource_mapping {"someProp": "lm_property"} + include_metadata true + metadata_to_exclude ["meta1", "meta3","someProp"] + ]).instance + tag = "lm.test" + time = Time.parse("2020-08-23T00:53:15+00:00").to_i + record = {"message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z" , + "meta1" => "testMeta1" , + "meta2" => "testMeta2", + "meta3" => "testMeta3", + "meta4" => "testMeta4", + "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, + "someProp" => "resourcePropVal" } + + result = plugin.process_record(tag, time, record) + + expected = { + "message" => "Hello from test", + "timestamp" => "2020-10-30T00:29:08.629701504Z", + "_lm.resourceId" => {"lm_property" => "resourcePropVal"}, + "meta2" => "testMeta2", + "meta4" => "testMeta4", + "meta5" => {"key1" => "value1", "key2" => {"key2_1" => "value2_1"}}, + } + assert_equal expected, result + end end end \ No newline at end of file