added validation for docker log format for recombine statment

adjust error logs to send instead of drop
coralogix · May 16, 2024 · 004a09d · 004a09d
1 parent 8b4fc44
commit 004a09d
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 5 deletions.
diff --git a/modules/ecs-ec2/otel_config.tftpl.yaml b/modules/ecs-ec2/otel_config.tftpl.yaml
@@ -21,26 +21,37 @@ receivers:
     include_file_path: true
     # add log.file.path to resource attributes
     operators:
+      - type: router
+        id: docker_log_json_parser
+        routes:
+          - output: json_parser
+            expr: 'body matches "^\\{\"log\".*\\}"'
+        default: move_log_file_path
+
       - type: json_parser
         parse_from: body
         parse_to: body
+        output: recombine
         timestamp:
           parse_from: body.time
           layout: '%Y-%m-%dT%H:%M:%S.%fZ'
+        default: move_log_file_path
 
+      # handle logs split by docker
       - type: recombine
         id: recombine
+        output: move_log_file_path
         combine_field: body.log
         source_identifier: attributes["log.file.path"]
         is_last_entry: body.log endsWith "\n"
-        force_flush_period: 10s
-        on_error: drop
+        force_flush_period: 10s	
+        on_error: send
         combine_with: ""
 
       - type: move
+        id: move_log_file_path
         from: attributes["log.file.path"]
         to: resource["log.file.path"]
-
 processors:
   resourcedetection:
     # ecs & docker detectors not required when using ecslogresourcedetection for logs

diff --git a/modules/ecs-ec2/otel_config_metrics.tftpl.yaml b/modules/ecs-ec2/otel_config_metrics.tftpl.yaml
@@ -21,23 +21,35 @@ receivers:
     include_file_path: true
     # add log.file.path to resource attributes
     operators:
+      - type: router
+        id: docker_log_json_parser
+        routes:
+          - output: json_parser
+            expr: 'body matches "^\\{\"log\".*\\}"'
+        default: move_log_file_path
+
       - type: json_parser
         parse_from: body
         parse_to: body
+        output: recombine
         timestamp:
           parse_from: body.time
           layout: '%Y-%m-%dT%H:%M:%S.%fZ'
+        default: move_log_file_path
 
+      # handle logs split by docker
       - type: recombine
         id: recombine
+        output: move_log_file_path
         combine_field: body.log
         source_identifier: attributes["log.file.path"]
         is_last_entry: body.log endsWith "\n"
-        force_flush_period: 10s
-        on_error: drop
+        force_flush_period: 10s	
+        on_error: send
         combine_with: ""
 
       - type: move
+        id: move_log_file_path
         from: attributes["log.file.path"]
         to: resource["log.file.path"]