Add:HTTP header tagging with DD_TRACE_HEADER_TAGS for clients

lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -1,5 +1,6 @@
 require 'uri'
 
+require_relative '../../../core/utils/hash'
 require_relative '../../metadata/ext'
 require_relative '../../propagation/http'
 require_relative 'ext'
@@ -58,6 +59,12 @@ def complete
                     set_span_error_message("Request has failed with HTTP error: #{response_code}")
                   end
                 end
+
+                @datadog_span.set_tags(
+                  Datadog.configuration.tracing.header_tags.response_tags(
+                    Core::Utils::Hash::CaseInsensitiveWrapper.new(parse_response_headers)
+                  )
+                )
               ensure
                 @datadog_span.finish
                 @datadog_span = nil
@@ -142,6 +149,14 @@ def datadog_tag_request
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, uri.host)
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, uri.port)
               span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, uri.host)
+
+              if @datadog_original_headers
+                span.set_tags(
+                  Datadog.configuration.tracing.header_tags.request_tags(
+                    Core::Utils::Hash::CaseInsensitiveWrapper.new(@datadog_original_headers)
+                  )
+                )
+              end
             end
 
             def set_span_error_message(message)
@@ -168,6 +183,26 @@ def analytics_enabled?
             def analytics_sample_rate
               datadog_configuration[:analytics_sample_rate]
             end
+
+            # `#response_headers` returns a "\n" concatenated String containing:
+            # * The HTTP Status-Line.
+            # * The response headers.
+            # * A trailing "\n".
+            #
+            # This method extracts only the headers from it.
+            def parse_response_headers
+              return {} if response_headers.empty?
+
+              lines = response_headers.split("\n")
+
+              lines = lines[1..(lines.size - 1)] # Remove Status-Line and trailing whitespace.
+
+              # Find only well-behaved HTTP headers.
+              lines.map do |line|
+                header = line.split(':', 2)
+                header.size != 2 ? nil : header
+              end.compact.to_h
+            end
           end
         end
       end

lib/datadog/tracing/contrib/excon/middleware.rb

@@ -133,6 +133,12 @@ def annotate!(span, datum)
             span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, datum[:method].to_s.upcase)
             span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, datum[:host])
             span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, datum[:port])
+
+            span.set_tags(
+              Datadog.configuration.tracing.header_tags.request_tags(
+                Core::Utils::Hash::CaseInsensitiveWrapper.new(datum[:headers])
+              )
+            )
           end
 
           def handle_response(datum)
@@ -144,6 +150,12 @@ def handle_response(datum)
                   response = datum[:response]
                   span.set_error(["Error #{response[:status]}", response[:body]]) if error_handler.call(response)
                   span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, response[:status])
+
+                  span.set_tags(
+                    Datadog.configuration.tracing.header_tags.response_tags(
+                      Core::Utils::Hash::CaseInsensitiveWrapper.new(response[:headers])
+                    )
+                  )
                 end
                 span.set_error(datum[:error]) if datum.key?(:error)
                 span.finish

lib/datadog/tracing/contrib/faraday/middleware.rb

@@ -61,12 +61,20 @@ def annotate!(span, env, options)
             span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, env[:method].to_s.upcase)
             span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, env[:url].host)
             span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, env[:url].port)
+
+            span.set_tags(
+              Datadog.configuration.tracing.header_tags.request_tags(env[:request_headers])
+            )
           end
 
           def handle_response(span, env, options)
             span.set_error(["Error #{env[:status]}", env[:body]]) if options.fetch(:error_handler).call(env)
 
             span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, env[:status])
+
+            span.set_tags(
+              Datadog.configuration.tracing.header_tags.response_tags(env[:response_headers])
+            )
           end
 
           def propagate!(trace, span, env)

lib/datadog/tracing/contrib/http/instrumentation.rb

@@ -89,6 +89,10 @@ def annotate_span_with_request!(span, request, request_options)
 
               # Set analytics sample rate
               set_analytics_sample_rate(span, request_options)
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.request_tags(request)
+              )
             end
 
             def annotate_span_with_response!(span, response, request_options)
@@ -97,6 +101,10 @@ def annotate_span_with_response!(span, response, request_options)
               span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, response.code)
 
               span.set_error(response) if request_options[:error_status_codes].include? response.code.to_i
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.response_tags(response)
+              )
             end
 
             def annotate_span_with_error!(span, error)

lib/datadog/tracing/contrib/httpclient/instrumentation.rb

@@ -71,6 +71,10 @@ def annotate_span_with_request!(span, req, req_options)
               span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, uri.host)
 
               set_analytics_sample_rate(span, req_options)
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.request_tags(req.http_header)
+              )
             end
 
             def annotate_span_with_response!(span, response, request_options)
@@ -81,6 +85,10 @@ def annotate_span_with_response!(span, response, request_options)
               if request_options[:error_status_codes].include? response.code.to_i
                 span.set_error(["Error #{response.status}", response.body])
               end
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.response_tags(response.header)
+              )
             end
 
             def annotate_span_with_error!(span, error)

lib/datadog/tracing/contrib/httprb/instrumentation.rb

@@ -79,6 +79,10 @@ def annotate_span_with_request!(span, req, req_options)
               end
 
               set_analytics_sample_rate(span, req_options)
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.request_tags(req.headers)
+              )
             end
 
             def annotate_span_with_response!(span, response, request_options)
@@ -91,6 +95,10 @@ def annotate_span_with_response!(span, response, request_options)
                 # parsing the response body message will alter downstream application behavior
                 span.set_error(["Error #{response.code}", 'Error'])
               end
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.response_tags(response.headers)
+              )
             end
 
             def annotate_span_with_error!(span, error)

lib/datadog/tracing/contrib/rest_client/request_patch.rb

@@ -53,6 +53,12 @@ def datadog_tag_request(uri, span)
               span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, method.to_s.upcase)
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, uri.host)
               span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, uri.port)
+
+              span.set_tags(
+                Datadog.configuration.tracing.header_tags.request_tags(
+                  Core::Utils::Hash::CaseInsensitiveWrapper.new(processed_headers)
+                )
+              )
             end
 
             def datadog_trace_request(uri)
@@ -71,6 +77,12 @@ def datadog_trace_request(uri)
                 # If so, add additional tags.
                 if response.is_a?(::RestClient::Response)
                   span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, response.code)
+
+                  span.set_tags(
+                    Datadog.configuration.tracing.header_tags.response_tags(
+                      Core::Utils::Hash::CaseInsensitiveWrapper.new(response.net_http_res.to_hash)
+                    )
+                  )
                 end
               end
             rescue ::RestClient::ExceptionWithResponse => e

spec/datadog/tracing/contrib/ethon/integration_test_spec.rb

@@ -8,6 +8,7 @@
 require 'datadog/tracing/contrib/ethon/multi_patch'
 require 'datadog/tracing/contrib/ethon/shared_examples'
 require 'datadog/tracing/contrib/support/spec_helper'
+require 'datadog/tracing/contrib/support/http'
 
 require 'spec/datadog/tracing/contrib/ethon/support/thread_helpers'
 
@@ -70,15 +71,35 @@
       easy = EthonSupport.ethon_easy_new(url: url)
       easy.customrequest = 'GET'
       easy.set_attributes(timeout_ms: timeout * 1000)
-      easy.headers = { key: 'value' }
+      easy.headers = request_headers
       easy.perform
       # Use Typhoeus response wrapper to simplify tests
       Typhoeus::Response.new(easy.mirror.options)
     end
 
+    let(:request_headers) { { key: 'value' } }
+
     it_behaves_like 'instrumented request' do
       let(:method) { 'N/A' }
     end
+
+    context 'when configured with global tag headers' do
+      include_context 'integration context'
+
+      before { request }
+
+      let(:request_headers) { { 'Request-Id' => 'test-request' } }
+
+      include_examples 'with request tracer header tags' do
+        let(:request_header_tag) { 'request-id' }
+        let(:request_header_tag_value) { 'test-request' }
+      end
+
+      include_examples 'with response tracer header tags' do
+        let(:response_header_tag) { 'connection' }
+        let(:response_header_tag_value) { 'Keep-Alive' }
+      end
+    end
   end
 
   context 'with single Multi request' do

spec/datadog/tracing/contrib/excon/instrumentation_spec.rb

@@ -3,6 +3,7 @@
 require 'datadog/tracing/contrib/analytics_examples'
 require 'datadog/tracing/contrib/environment_service_name_examples'
 require 'datadog/tracing/contrib/span_attribute_schema_examples'
+require 'datadog/tracing/contrib/support/http'
 
 require 'excon'
 require 'ddtrace'
@@ -12,7 +13,7 @@
   let(:connection_options) { { mock: true } }
   let(:connection) do
     Excon.new('http://example.com', connection_options).tap do
-      Excon.stub({ method: :get, path: '/success' }, body: 'OK', status: 200)
+      Excon.stub({ method: :get, path: '/success' }, body: 'OK', status: 200, headers: response_headers)
       Excon.stub({ method: :post, path: '/failure' }, body: 'Boom!', status: 500)
       Excon.stub({ method: :get, path: '/not_found' }, body: 'Not Found.', status: 404)
       Excon.stub(
@@ -25,6 +26,7 @@
   end
   let(:middleware_options) { {} }
   let(:configuration_options) { {} }
+  let(:response_headers) { {} }
 
   let(:request_span) do
     spans.find { |span| span.name == Datadog::Tracing::Contrib::Excon::Ext::SPAN_REQUEST }
@@ -80,7 +82,8 @@
   end
 
   context 'when there is successful request' do
-    subject!(:response) { connection.get(path: '/success') }
+    subject!(:response) { connection.get(path: '/success', headers: request_headers) }
+    let(:request_headers) { {} }
 
     it_behaves_like 'environment service name', 'DD_TRACE_EXCON_SERVICE_NAME'
     it_behaves_like 'schema version span'
@@ -114,6 +117,22 @@
     it_behaves_like 'a peer service span' do
       let(:peer_hostname) { 'example.com' }
     end
+
+    context 'when configured with global tag headers' do
+      let(:header_span) { request_span }
+      let(:request_headers) { { 'Request-Id' => 'test-request' } }
+      let(:response_headers) { { 'Response-Id' => 'test-response' } }
+
+      include_examples 'with request tracer header tags' do
+        let(:request_header_tag) { 'request-id' }
+        let(:request_header_tag_value) { 'test-request' }
+      end
+
+      include_examples 'with response tracer header tags' do
+        let(:response_header_tag) { 'response-id' }
+        let(:response_header_tag_value) { 'test-response' }
+      end
+    end
   end
 
   context 'when there is a failing request' do

spec/datadog/tracing/contrib/faraday/middleware_spec.rb

@@ -3,6 +3,7 @@
 require 'datadog/tracing/contrib/analytics_examples'
 require 'datadog/tracing/contrib/environment_service_name_examples'
 require 'datadog/tracing/contrib/span_attribute_schema_examples'
+require 'datadog/tracing/contrib/support/http'
 
 require 'faraday'
 
@@ -14,7 +15,7 @@
     ::Faraday.new('http://example.com') do |builder|
       builder.use(:ddtrace, middleware_options) if use_middleware
       builder.adapter(:test) do |stub|
-        stub.get('/success') { |_| [200, {}, 'OK'] }
+        stub.get('/success') { |_| [200, response_headers, 'OK'] }
         stub.post('/failure') { |_| [500, {}, 'Boom!'] }
         stub.get('/not_found') { |_| [404, {}, 'Not Found.'] }
         stub.get('/error') { |_| raise ::Faraday::ConnectionFailed, 'Test error' }
@@ -25,6 +26,7 @@
   let(:use_middleware) { true }
   let(:middleware_options) { {} }
   let(:configuration_options) { {} }
+  let(:response_headers) { {} }
 
   before do
     Datadog.configure do |c|
@@ -40,7 +42,8 @@
   end
 
   context 'without explicit middleware configured' do
-    subject(:response) { client.get('/success') }
+    subject(:response) { client.get('/success', nil, request_headers) }
+    let(:request_headers) { {} }
 
     let(:use_middleware) { false }
 
@@ -75,6 +78,23 @@
       expect { response }.to_not output(/WARNING/).to_stderr
     end
 
+    context 'when configured with global tag headers' do
+      before { response }
+
+      let(:request_headers) { { 'Request-Id' => 'test-request' } }
+      let(:response_headers) { { 'Response-Id' => 'test-response' } }
+
+      include_examples 'with request tracer header tags' do
+        let(:request_header_tag) { 'request-id' }
+        let(:request_header_tag_value) { 'test-request' }
+      end
+
+      include_examples 'with response tracer header tags' do
+        let(:response_header_tag) { 'response-id' }
+        let(:response_header_tag_value) { 'test-response' }
+      end
+    end
+
     context 'with default Faraday connection' do
       subject(:response) { client.get('http://example.com/success') }