Rescue instrumentation errors

DataDog · Sep 25, 2024 · 24eebe9 · 24eebe9
1 parent 2209a7c
commit 24eebe9
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 2 deletions.
diff --git a/lib/datadog/tracing/contrib/active_record/events/instantiation.rb b/lib/datadog/tracing/contrib/active_record/events/instantiation.rb
@@ -4,6 +4,7 @@
 require_relative '../../analytics'
 require_relative '../ext'
 require_relative '../event'
+require_relative '../../../../core/telemetry/logger'
 
 module Datadog
   module Tracing
@@ -48,7 +49,8 @@ def on_start(span, event, _id, payload)
               span.set_tag(Ext::TAG_INSTANTIATION_CLASS_NAME, payload.fetch(:class_name))
               span.set_tag(Ext::TAG_INSTANTIATION_RECORD_COUNT, payload.fetch(:record_count))
             rescue StandardError => e
-              Datadog.logger.debug(e.message)
+              Datadog.logger.error(e.message)
+              Datadog::Core::Telemetry::Logger.report(e)
             end
           end
         end

diff --git a/lib/datadog/tracing/contrib/active_record/events/sql.rb b/lib/datadog/tracing/contrib/active_record/events/sql.rb
@@ -6,6 +6,7 @@
 require_relative '../ext'
 require_relative '../../analytics'
 require_relative '../../utils/database'
+require_relative '../../../../core/telemetry/logger'
 
 module Datadog
   module Tracing
@@ -68,7 +69,8 @@ def on_start(span, event, _id, payload)
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, config[:host]) if config[:host]
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, config[:port]) if config[:port]
             rescue StandardError => e
-              Datadog.logger.debug(e.message)
+              Datadog.logger.error(e.message)
+              Datadog::Core::Telemetry::Logger.report(e)
             end
           end
         end

diff --git a/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb b/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb
@@ -2,6 +2,7 @@
 
 require_relative '../../ext'
 require_relative '../event'
+require_relative '../../../../../core/telemetry/logger'
 
 module Datadog
   module Tracing
@@ -81,6 +82,9 @@ def on_start(span, event, _id, payload)
                 span.set_tag('EVENT', event)
 
                 set_cache_key(span, key, mapping[:multi_key])
+              rescue StandardError => e
+                Datadog.logger.error(e.message)
+                Datadog::Core::Telemetry::Logger.report(e)
               end
 
               def set_cache_key(span, key, multi_key)

diff --git a/lib/datadog/tracing/contrib/aws/instrumentation.rb b/lib/datadog/tracing/contrib/aws/instrumentation.rb
@@ -29,6 +29,7 @@ def call(context)
           private
 
           # rubocop:disable Metrics/AbcSize
+          # rubocop:disable Metrics/MethodLength
           def annotate!(span, context)
             span.service = configuration[:service_name]
             span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
@@ -76,7 +77,11 @@ def annotate!(span, context)
             span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, context.safely(:status_code))
 
             Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
+          rescue StandardError => e
+            Datadog.logger.error(e.message)
+            Datadog::Core::Telemetry::Logger.report(e)
           end
+          # rubocop:enable Metrics/MethodLength
           # rubocop:enable Metrics/AbcSize
 
           def configuration

diff --git a/lib/datadog/tracing/contrib/faraday/middleware.rb b/lib/datadog/tracing/contrib/faraday/middleware.rb
@@ -7,6 +7,7 @@
 require_relative '../analytics'
 require_relative 'ext'
 require_relative '../http_annotation_helper'
+require_relative '../../../core/telemetry/logger'
 
 module Datadog
   module Tracing
@@ -37,6 +38,7 @@ def call(env)
 
           attr_reader :app
 
+          # rubocop:disable Metrics/AbcSize
           def annotate!(span, env, options)
             span.resource = resource_name(env)
             span.service = service_name(env[:url].host, options)
@@ -75,7 +77,11 @@ def annotate!(span, env, options)
             )
 
             Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
+          rescue StandardError => e
+            Datadog.logger.error(e.message)
+            Datadog::Core::Telemetry::Logger.report(e)
           end
+          # rubocop:enable Metrics/AbcSize
 
           def handle_response(span, env, options)
             span.set_error(["Error #{env[:status]}", env[:body]]) if options[:error_status_codes].include? env[:status]
@@ -85,6 +91,9 @@ def handle_response(span, env, options)
             span.set_tags(
               Datadog.configuration.tracing.header_tags.response_tags(env[:response_headers])
             )
+          rescue StandardError => e
+            Datadog.logger.error(e.message)
+            Datadog::Core::Telemetry::Logger.report(e)
           end
 
           def propagate!(trace, span, env)

diff --git a/lib/datadog/tracing/contrib/httpclient/instrumentation.rb b/lib/datadog/tracing/contrib/httpclient/instrumentation.rb
@@ -102,6 +102,9 @@ def annotate_span_with_response!(span, response, request_options)
               span.set_tags(
                 Datadog.configuration.tracing.header_tags.response_tags(response.header)
               )
+            rescue StandardError => e
+              Datadog.logger.error("error preparing span from httpclient response: #{e}, Source: #{e.backtrace}")
+              Datadog::Core::Telemetry::Logger.report(e)
             end
 
             def annotate_span_with_error!(span, error)

diff --git a/lib/datadog/tracing/contrib/httprb/instrumentation.rb b/lib/datadog/tracing/contrib/httprb/instrumentation.rb
@@ -110,6 +110,9 @@ def annotate_span_with_response!(span, response, request_options)
               span.set_tags(
                 Datadog.configuration.tracing.header_tags.response_tags(response.headers)
               )
+            rescue StandardError => e
+              logger.error("error preparing span from http.rb response: #{e}, Source: #{e.backtrace}")
+              Datadog::Core::Telemetry::Logger.report(e)
             end
 
             def annotate_span_with_error!(span, error)

diff --git a/lib/datadog/tracing/contrib/mongodb/subscribers.rb b/lib/datadog/tracing/contrib/mongodb/subscribers.rb
@@ -70,6 +70,8 @@ def started(event)
 
             # set the resource with the quantized query
             span.resource = serialized_query
+          rescue StandardError => e
+            Datadog.logger.debug("error when handling MongoDB 'started' event: #{e}")
           end
           # rubocop:enable Metrics/AbcSize
 

diff --git a/lib/datadog/tracing/contrib/redis/tags.rb b/lib/datadog/tracing/contrib/redis/tags.rb
@@ -4,6 +4,7 @@
 require_relative '../analytics'
 require_relative 'ext'
 require_relative '../ext'
+require_relative '../../../core/telemetry/logger'
 
 module Datadog
   module Tracing
@@ -45,6 +46,9 @@ def set_common_tags(client, span, raw_command)
               span.set_tag Ext::TAG_RAW_COMMAND, raw_command
 
               Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
+            rescue StandardError => e
+              Datadog.logger.error(e.message)
+              Datadog::Core::Telemetry::Logger.report(e)
             end
 
             private