Merge pull request #30 from logicmonitor/enforce_encoding-utf-8

Enforce encoding utf 8

README.md

@@ -67,4 +67,5 @@ See the [LogicMonitor Helm repository](https://github.com/logicmonitor/k8s-helm-
 | `access_id` | LM API Token access ID. |
 | `access_key` | LM API Token access key. |
 | `flush_interval` | Defines the time in seconds to wait before sending batches of logs to LogicMonitor. Default is `60s`. |
-| `debug` | When `true`, logs more information to the fluentd console. |
+| `debug` | When `true`, logs more information to the fluentd console. |
+| `force_encoding` | Specify charset when logs contains invalid utf-characters. |

fluent-plugin-lm-logs.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs"
-  spec.version                        = '0.0.9'
+  spec.version                        = '0.0.10'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "rubygems@logicmonitor.com"
   spec.summary                        = "LogicMonitor logs fluentd output plugin"

lib/fluent/plugin/out_lm.rb

@@ -25,6 +25,8 @@ class LmOutput < BufferedOutput
     config_param :resource_mapping,  :hash, :default => {"host": "system.hostname", "hostname": "system.hostname"}
 
     config_param :debug,  :bool, :default => false
+
+    config_param :force_encoding,  :string, :default => ""
 
     # This method is called before starting.
     # 'conf' is a Hash that includes configuration parameters.
@@ -71,6 +73,11 @@ def process_record(tag, time, record)
       resource_map = {}
       lm_event = {}
       lm_event["message"] = record["message"]
+
+      if @force_encoding != ""
+        lm_event["message"] = lm_event["message"].force_encoding(@force_encoding).encode("UTF-8")
+      end
+
       if record["_lm.resourceId"] == nil
           @resource_mapping.each do |key, value|
             k = value

test/plugin/test_out_lm.rb

@@ -61,4 +61,21 @@ def create_valid_subject
         assert_equal expected, result
       end
   end
+
+  sub_test_case "force_encoding" do
+      test "force_encoding passed as true, should convert invalid utf-8 characters" do
+        plugin = create_driver(%[
+          resource_mapping {"a.b": "lm_property"} 
+          force_encoding ISO-8859-1
+      ]).instance
+
+      tag = "lm.test"
+      time = Time.parse("2020-08-23T00:53:15+00:00").to_i
+      event = {"message" => "LogicMonitor\xAE", "a" => { "b" => "lm_property_value" } }
+
+      event = plugin.process_record(tag, time, event)
+
+      assert_equal "LogicMonitor®", event["message"]
+    end
+  end
 end