extract schema at teh end of the request life cycle

lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -75,6 +75,16 @@ def call(env)
 
             _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
 
+            result = scope.processor_context.extract_schema
+
+            if result
+              scope.processor_context.events << {
+                trace: scope.trace,
+                span: scope.service_entry_span,
+                waf_result: result,
+              }
+            end
+
             scope.processor_context.events.each do |e|
               e[:response] ||= gateway_response
               e[:request]  ||= gateway_request

lib/datadog/appsec/event.rb

@@ -62,6 +62,7 @@ def self.record_via_span(span, *events)
 
           # prepare and gather tags to apply
           service_entry_tags = build_service_entry_tags(event_group)
+
           # complex types are unsupported, we need to serialize to a string
           triggers = service_entry_tags.delete('_dd.appsec.triggers')
           span.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
@@ -104,8 +105,15 @@ def self.build_service_entry_tags(event_group)
           tags['_dd.origin'] = 'appsec'
 
           # accumulate triggers
+          waf_result = event[:waf_result]
           tags['_dd.appsec.triggers'] ||= []
-          tags['_dd.appsec.triggers'] += event[:waf_result].events
+          tags['_dd.appsec.triggers'] += waf_result.events
+
+          waf_result.derivatives.each do |key, value|
+            tags[key] = JSON.dump(value)
+          end
+
+          tags
         end
       end
     end

lib/datadog/appsec/processor.rb

@@ -22,8 +22,6 @@ def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
 
           start_ns = Core::Utils::Time.get_time(:nanosecond)
 
-          # this WAF::Context#run call is not thread safe as it mutates the context
-          # TODO: remove multiple assignment
           _code, res = @context.run(input, timeout)
 
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
@@ -38,9 +36,34 @@ def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
           @run_mutex.unlock
         end
 
+        def extract_schema
+          return unless extract_schema?
+
+          @run_mutex.lock
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          res
+        ensure
+          @run_mutex.unlock
+        end
+
         def finalize
           @context.finalize
         end
+
+        private
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
+        end
       end
 
       attr_reader :diagnostics, :addresses

sig/datadog/appsec/processor.rbs

@@ -16,7 +16,11 @@ module Datadog
 
         def initialize: (Processor processor) -> void
         def run: (data input, ?::Integer timeout) -> WAF::Result
+        def extract_schema: () -> WAF::Result?
         def finalize: () -> void
+
+        private
+        def extract_schema?: () -> bool
       end
 
       def self.active_context: () -> Context

spec/datadog/appsec/contrib/rack/integration_test_spec.rb

@@ -23,6 +23,8 @@
   let(:appsec_ip_denylist) { [] }
   let(:appsec_user_id_denylist) { [] }
   let(:appsec_ruleset) { :recommended }
+  let(:api_security_enabled) { false }
+  let(:api_security_sample) { 0.0 }
 
   let(:crs_942_100) do
     {
@@ -136,6 +138,8 @@
       c.appsec.ip_denylist = appsec_ip_denylist
       c.appsec.user_id_denylist = appsec_user_id_denylist
       c.appsec.ruleset = appsec_ruleset
+      c.appsec.api_security.enabled = api_security_enabled
+      c.appsec.api_security.sample_rate = api_security_sample
     end
   end
 
@@ -231,6 +235,19 @@
           it_behaves_like 'a GET 200 span'
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace without AppSec events'
+
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_ok }
+
+            it_behaves_like 'normal with tracing disable'
+            it_behaves_like 'a GET 200 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace without AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
         end
 
         context 'with an event-triggering request in headers' do
@@ -243,6 +260,20 @@
           it_behaves_like 'a GET 200 span'
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace with AppSec events'
+
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_ok }
+            it { expect(triggers).to be_a Array }
+
+            it_behaves_like 'normal with tracing disable'
+            it_behaves_like 'a GET 200 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace with AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
         end
 
         context 'with an event-triggering request in query string' do
@@ -264,6 +295,19 @@
             it_behaves_like 'a GET 403 span'
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_forbidden }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a GET 403 span'
+              it_behaves_like 'a trace with AppSec api security tags'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events', { blocking: true }
+            end
           end
         end
 
@@ -278,6 +322,19 @@
           it_behaves_like 'a GET 403 span'
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace with AppSec events'
+
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_forbidden }
+
+            it_behaves_like 'normal with tracing disable'
+            it_behaves_like 'a GET 403 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace with AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
         end
 
         context 'with an event-triggering response' do
@@ -302,6 +359,19 @@
             it_behaves_like 'a GET 403 span'
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_forbidden }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a GET 403 span'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events', { blocking: true }
+              it_behaves_like 'a trace with AppSec api security tags'
+            end
           end
         end
 
@@ -324,6 +394,19 @@
             it_behaves_like 'a GET 403 span'
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_forbidden }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a GET 403 span'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events', { blocking: true }
+              it_behaves_like 'a trace with AppSec api security tags'
+            end
           end
         end
       end
@@ -343,6 +426,19 @@
           it_behaves_like 'a POST 200 span'
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace without AppSec events'
+
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_ok }
+
+            it_behaves_like 'normal with tracing disable'
+            it_behaves_like 'a POST 200 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace without AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
         end
 
         context 'with an event-triggering request in application/x-www-form-url-encoded body' do
@@ -362,6 +458,18 @@
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace with AppSec events'
 
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_ok }
+
+            it_behaves_like 'a POST 200 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace with AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
+
           context 'and a blocking rule' do
             let(:appsec_ruleset) { crs_942_100 }
 
@@ -371,6 +479,19 @@
             it_behaves_like 'a POST 403 span'
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_forbidden }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a POST 403 span'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events', { blocking: true }
+              it_behaves_like 'a trace with AppSec api security tags'
+            end
           end
         end
 
@@ -394,6 +515,19 @@
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events'
 
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_ok }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a POST 200 span'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events'
+              it_behaves_like 'a trace with AppSec api security tags'
+            end
+
             context 'and a blocking rule' do
               let(:appsec_ruleset) { crs_942_100 }
 
@@ -403,6 +537,19 @@
               it_behaves_like 'a POST 403 span'
               it_behaves_like 'a trace with AppSec tags'
               it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+              context 'with schema extraction' do
+                let(:api_security_enabled) { true }
+                let(:api_security_sample) { 1 }
+
+                it { is_expected.to be_forbidden }
+
+                it_behaves_like 'normal with tracing disable'
+                it_behaves_like 'a POST 403 span'
+                it_behaves_like 'a trace with AppSec tags'
+                it_behaves_like 'a trace with AppSec events', { blocking: true }
+                it_behaves_like 'a trace with AppSec api security tags'
+              end
             end
           end
         end
@@ -436,6 +583,19 @@
           it_behaves_like 'a trace with AppSec tags'
           it_behaves_like 'a trace with AppSec events'
 
+          context 'with schema extraction' do
+            let(:api_security_enabled) { true }
+            let(:api_security_sample) { 1 }
+
+            it { is_expected.to be_ok }
+
+            it_behaves_like 'normal with tracing disable'
+            it_behaves_like 'a POST 200 span'
+            it_behaves_like 'a trace with AppSec tags'
+            it_behaves_like 'a trace with AppSec events'
+            it_behaves_like 'a trace with AppSec api security tags'
+          end
+
           context 'and a blocking rule' do
             let(:appsec_ruleset) { crs_942_100 }
 
@@ -445,6 +605,19 @@
             it_behaves_like 'a POST 403 span'
             it_behaves_like 'a trace with AppSec tags'
             it_behaves_like 'a trace with AppSec events', { blocking: true }
+
+            context 'with schema extraction' do
+              let(:api_security_enabled) { true }
+              let(:api_security_sample) { 1 }
+
+              it { is_expected.to be_forbidden }
+
+              it_behaves_like 'normal with tracing disable'
+              it_behaves_like 'a POST 403 span'
+              it_behaves_like 'a trace with AppSec tags'
+              it_behaves_like 'a trace with AppSec events', { blocking: true }
+              it_behaves_like 'a trace with AppSec api security tags'
+            end
           end
         end
       end