Merge branch 'master' into graphql-fix

.env

@@ -33,7 +33,7 @@ TEST_POSTGRES_HOST=127.0.0.1
 TEST_POSTGRES_PASSWORD=postgres
 TEST_POSTGRES_PORT=5432
 TEST_POSTGRES_USER=postgres
-TEST_PRESTO_HOST=presto
+TEST_PRESTO_HOST=127.0.0.1
 TEST_PRESTO_PORT=8080
 TEST_REDIS_HOST=127.0.0.1
 TEST_REDIS_PORT=6379

.github/workflows/add-milestone-to-pull-requests.yml

@@ -5,6 +5,9 @@ on:
 
 jobs:
   add_milestone:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     if: github.event.pull_request.merged == true && github.event.pull_request.milestone == null
     steps:

.github/workflows/system-tests.yml

@@ -28,13 +28,16 @@ jobs:
           - name: proxy
             internal: datadog/system-tests:proxy-v1
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
     name: Build (${{ matrix.image.name }})
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
           ref: ${{ env.ST_REF }}
+          persist-credentials: false
       - name: Pull released image
         run: |
           if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then
@@ -100,18 +103,22 @@ jobs:
           - graphql23
     runs-on: ubuntu-latest
     name: Build (${{ matrix.app }})
+    permissions:
+      packages: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
           ref: ${{ env.ST_REF }}
+          persist-credentials: false
       - name: Checkout ${{ matrix.library.repository }}
         uses: actions/checkout@v4
         with:
           repository: '${{ matrix.library.repository }}'
           path: 'binaries/${{ matrix.library.path }}'
           fetch-depth: 2
+          persist-credentials: false
       - name: Pull released image
         run: |
           if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then
@@ -252,6 +259,7 @@ jobs:
         with:
           repository: 'DataDog/system-tests'
           ref: ${{ env.ST_REF }}
+          persist-credentials: false
       - name: Pull runner image
         run: |
           docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }}
@@ -304,6 +312,8 @@ jobs:
           - graphql23
     runs-on: ubuntu-latest
     needs:
+      - build-harness
+      - build-apps
       - test
     if: ${{ always() }}
     name: Aggregate (${{ matrix.app }})
@@ -317,6 +327,7 @@ jobs:
         with:
           repository: 'DataDog/system-tests'
           ref: ${{ env.ST_REF }}
+          persist-credentials: false
       - name: Retrieve logs
         uses: actions/download-artifact@v4
         with:

.github/workflows/test-memory-leaks.yaml

@@ -0,0 +1,33 @@
+name: Test for memory leaks
+on: [push]
+jobs:
+  test-memcheck:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.4.0-preview1 # TODO: Use stable version once 3.4 is out
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+          bundler: latest
+          cache-version: v1 # bump this to invalidate cache
+      - run: sudo apt update && sudo apt install -y valgrind && valgrind --version
+      - run: bundle exec rake compile spec:profiling:memcheck
+  test-asan:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      # We're using a fork of ruby/setup-ruby because the "asan" tool is built into the clang compiler toolchain, and
+      # needs Ruby to be built with a special configuration.
+      #
+      # The special configuration is not yet available in the upstream `ruby/setup-ruby` github action, so I needed to
+      # fork it and push a small tweak to make it available.
+      #
+      # (The Ruby builds were added in https://github.com/ruby/ruby-dev-builder/pull/10 ).
+      - uses: datadog/setup-ruby@0c7206d6db81faf999795ceebfac00d164298bd5
+        with:
+          ruby-version: asan
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+          bundler: latest
+          cache-version: v1 # bump this to invalidate cache
+      - run: env RUBY_FREE_AT_EXIT=1 LSAN_OPTIONS=verbosity=0:log_threads=1:suppressions=`pwd`/suppressions/lsan.supp ASAN_OPTIONS=detect_leaks=1 bundle exec rake spec:profiling:main

.github/workflows/update-gemfiles.yml

@@ -29,6 +29,9 @@ jobs:
   check:
     name: Update Gemfiles
     runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+      pull-requests: write
     steps:
       # Only execute if there's a PR attached to this branch.
       # Because we execute on `push`, we have to double check here if this is part of a PR.

.rubocop.yml

@@ -24,6 +24,8 @@ AllCops:
     - 'spec/**/**/interesting_backtrace_helper.rb' # This file needs quite a few bizarre code patterns by design
     - 'vendor/bundle/**/*'
     - 'spec/datadog/tracing/contrib/grpc/support/gen/**/*.rb' # Skip protoc autogenerated code
+    - lib/datadog/di/**/*
+    - spec/datadog/di/**/*
   NewCops: disable # Don't allow new cops to be enabled implicitly.
   SuggestExtensions: false # Stop pushing suggestions constantly.
 

CHANGELOG.md

@@ -2,6 +2,34 @@
 
 ## [Unreleased]
 
+## [2.3.0] - 2024-08-22
+
+### Added
+
+* Core: Support agentless telemetry ([#3779][])
+* Tracing: Add support for span events ([#3776][])
+* Tracing: Add tags to enable inferred service dependencies for databases ([#3789][])
+* Tracing: Emit log message and instructions for incompatible Lograge setup ([#3812][], [#3839][])
+* Tracing: Add `append_comment` option to append SQL comment propagation for `mysql`, `pg` and `trilogy` ([#3809][])
+* AppSec: Add threat detection and protection for `graphql` ([#3769][], [#3814][])
+
+### Changed
+
+* Core: Enable crashtracking by default ([#3826][])
+* Profiling: Reduce allocation overhead ([#3805][], [#3797][])
+* Profiling: Speed up stack sampling ([#3837][])
+* Profiling: Upgrade to libdatadog 11 ([#3799][])
+* Profiling: Disable allocation counting feature by default ([#3798][])
+* Profiling: Reduce the maximum biased result for allocation samples ([#3793][])
+* Tracing: Reduce noisy integration logs ([#3785][])
+
+### Fixed
+
+* Tracing: Fix `require` issue for `graphql` ([#3813][])
+* AppSec: Fix an error when parsing http headers with integer value ([#3790][])
+* AppSec: Fix an error when tracking login failure without `user_id` ([#3841][])
+* Fix a syntax error for Ruby < 2.4 during single step instrumentation ([#3795][])
+
 ## [2.2.0] - 2024-07-11
 
 ### Added
@@ -2934,7 +2962,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.2.0...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v2.3.0...master
+[2.3.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.2.0...v2.3.0
 [2.2.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.1.0...v2.2.0
 [2.1.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0...v2.1.0
 [2.0.0]: https://github.com/DataDog/dd-trace-rb/compare/v2.0.0.rc1...v2.0.0
@@ -4331,9 +4360,29 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#3753]: https://github.com/DataDog/dd-trace-rb/issues/3753
 [#3757]: https://github.com/DataDog/dd-trace-rb/issues/3757
 [#3759]: https://github.com/DataDog/dd-trace-rb/issues/3759
+[#3769]: https://github.com/DataDog/dd-trace-rb/issues/3769
 [#3770]: https://github.com/DataDog/dd-trace-rb/issues/3770
 [#3772]: https://github.com/DataDog/dd-trace-rb/issues/3772
 [#3774]: https://github.com/DataDog/dd-trace-rb/issues/3774
+[#3776]: https://github.com/DataDog/dd-trace-rb/issues/3776
+[#3779]: https://github.com/DataDog/dd-trace-rb/issues/3779
+[#3785]: https://github.com/DataDog/dd-trace-rb/issues/3785
+[#3789]: https://github.com/DataDog/dd-trace-rb/issues/3789
+[#3790]: https://github.com/DataDog/dd-trace-rb/issues/3790
+[#3793]: https://github.com/DataDog/dd-trace-rb/issues/3793
+[#3795]: https://github.com/DataDog/dd-trace-rb/issues/3795
+[#3797]: https://github.com/DataDog/dd-trace-rb/issues/3797
+[#3798]: https://github.com/DataDog/dd-trace-rb/issues/3798
+[#3799]: https://github.com/DataDog/dd-trace-rb/issues/3799
+[#3805]: https://github.com/DataDog/dd-trace-rb/issues/3805
+[#3809]: https://github.com/DataDog/dd-trace-rb/issues/3809
+[#3812]: https://github.com/DataDog/dd-trace-rb/issues/3812
+[#3813]: https://github.com/DataDog/dd-trace-rb/issues/3813
+[#3814]: https://github.com/DataDog/dd-trace-rb/issues/3814
+[#3826]: https://github.com/DataDog/dd-trace-rb/issues/3826
+[#3837]: https://github.com/DataDog/dd-trace-rb/issues/3837
+[#3839]: https://github.com/DataDog/dd-trace-rb/issues/3839
+[#3841]: https://github.com/DataDog/dd-trace-rb/issues/3841
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot
@@ -4485,4 +4534,4 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [@y-yagi]: https://github.com/y-yagi
 [@yujideveloper]: https://github.com/yujideveloper
 [@yukimurasawa]: https://github.com/yukimurasawa
-[@zachmccormick]: https://github.com/zachmccormick
+[@zachmccormick]: https://github.com/zachmccormick

Gemfile

@@ -44,7 +44,12 @@ gem 'webmock', '>= 3.10.0'
 
 gem 'rexml', '>= 3.2.7' # https://www.ruby-lang.org/en/news/2024/05/16/dos-rexml-cve-2024-35176/
 
-gem 'webrick', '>= 1.7.0' if RUBY_VERSION >= '3.0.0' # No longer bundled by default since Ruby 3.0
+if RUBY_VERSION.start_with?('3.4.')
+  # ruby 3.4 breaks stable webrick; we need this fix until a version later than 1.8.1 comes out
+  gem 'webrick', git: 'https://github.com/ruby/webrick.git', ref: '0c600e169bd4ae267cb5eeb6197277c848323bbe'
+elsif RUBY_VERSION >= '3.0.0' # No longer bundled by default since Ruby 3.0
+  gem 'webrick', '>= 1.7.0'
+end
 
 gem 'yard', '~> 0.9' # NOTE: YardDoc is generated with ruby 3.2 in GitHub Actions
 
@@ -82,6 +87,7 @@ group :check do
     gem 'rbs', '~> 3.2.0', require: false
     gem 'steep', '~> 1.6.0', require: false
   end
+  gem 'ruby_memcheck', '>= 3' if RUBY_VERSION >= '3.4.0' && RUBY_PLATFORM != 'java'
   gem 'standard', require: false
 end
 

Matrixfile

@@ -164,7 +164,12 @@
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },
   'stripe' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'stripe-12' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'stripe-11' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'stripe-10' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'stripe-9'  => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'stripe-8'  => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'stripe-7'  => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
   },
   'sucker_punch' => {
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'

Rakefile

@@ -6,6 +6,18 @@ require 'rspec/core/rake_task'
 require 'rake/extensiontask'
 require 'yard'
 require 'os'
+if Gem.loaded_specs.key? 'ruby_memcheck'
+  require 'ruby_memcheck'
+  require 'ruby_memcheck/rspec/rake_task'
+
+  RubyMemcheck.config(
+    # If there's an error, print the suppression for that error, to allow us to easily skip such an error if it's
+    # a false-positive / something in the VM we can't fix.
+    valgrind_generate_suppressions: true,
+    # This feature provides better quality data -- I couldn't get good output out of ruby_memcheck without it.
+    use_only_ruby_free_at_exit: true,
+  )
+end
 
 Dir.glob('tasks/*.rake').each { |r| import r }
 
@@ -182,18 +194,11 @@ namespace :spec do
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:contrib) do |t, args|
     contrib_paths = [
-      'analytics',
-      'configurable',
+      '*',
       'configuration/*',
       'configuration/resolvers/*',
-      'extensions',
-      'integration',
-      'patchable',
-      'patcher',
-      'registerable',
-      'registry',
       'registry/*',
-      'propagation/**/*'
+      'propagation/**/*',
     ].join(',')
 
     t.pattern = "spec/**/contrib/{#{contrib_paths}}_spec.rb"
@@ -314,6 +319,25 @@ namespace :spec do
       t.rspec_opts = [*args.to_a, '-t ractors'].join(' ')
     end
 
+    desc 'Run spec:profiling:main tests with memory leak checking'
+    if Gem.loaded_specs.key?('ruby_memcheck')
+      RubyMemcheck::RSpec::RakeTask.new(:memcheck) do |t, args|
+        t.pattern = 'spec/datadog/profiling/**/*_spec.rb,spec/datadog/profiling_spec.rb'
+        # Some of our specs use multi-threading + busy looping, or multiple processes, or are just really really slow.
+        # We skip running these when running under valgrind.
+        # (As a reminder, by default valgrind simulates a sequential/single-threaded execution).
+        #
+        # @ivoanjo: I previously tried https://github.com/Shopify/ruby_memcheck/issues/51 but in some cases valgrind
+        # would give incomplete output, causing a "FATAL: Premature end of data in tag valgrindoutput line 3" error in
+        # ruby_memcheck. I did not figure out why exactly.
+        t.rspec_opts = [*args.to_a, '-t ~ractors -t ~memcheck_valgrind_skip'].join(' ')
+      end
+    else
+      task :memcheck do
+        raise 'Memcheck requires the ruby_memcheck gem to be installed'
+      end
+    end
+
     # Make sure each profiling test suite has a dependency on compiled native extensions
     Rake::Task[:all].prerequisite_tasks.each { |t| t.enhance([:compile_native_extensions]) }
   end

Steepfile

@@ -68,6 +68,7 @@ target :datadog do
   ignore 'lib/datadog/core/metrics/metric.rb'
   ignore 'lib/datadog/core/metrics/options.rb'
   ignore 'lib/datadog/core/pin.rb'
+  ignore 'lib/datadog/core/rate_limiter.rb'
   ignore 'lib/datadog/core/runtime/ext.rb'
   ignore 'lib/datadog/core/runtime/metrics.rb'
   ignore 'lib/datadog/core/transport/ext.rb'
@@ -495,7 +496,6 @@ target :datadog do
   ignore 'lib/datadog/tracing/sampling/priority_sampler.rb'
   ignore 'lib/datadog/tracing/sampling/rate_by_key_sampler.rb'
   ignore 'lib/datadog/tracing/sampling/rate_by_service_sampler.rb'
-  ignore 'lib/datadog/tracing/sampling/rate_limiter.rb'
   ignore 'lib/datadog/tracing/sampling/rate_sampler.rb'
   ignore 'lib/datadog/tracing/sampling/rule.rb'
   ignore 'lib/datadog/tracing/sampling/rule_sampler.rb'
@@ -569,6 +569,7 @@ target :datadog do
   library 'passenger'
   library 'webmock'
   library 'graphql'
+  library 'datadog-ci'
 
   # TODO: gem 'libddwaf'
   library 'libddwaf'

appraisal/jruby-9.2.rb

@@ -186,10 +186,15 @@
   gem 'http', '~> 4' # TODO: Fix test breakage and flakiness for 5+
   gem 'httpclient'
   gem 'rest-client'
-  gem 'stripe', '~> 7.0'
   gem 'typhoeus'
 end
 
+(7..12).each do |n|
+  appraise "stripe-#{n}" do
+    gem 'stripe', "~> #{n}"
+  end
+end
+
 [2, 3].each do |n|
   appraise "opensearch-#{n}" do
     gem 'opensearch-ruby', "~> #{n}"
@@ -209,7 +214,7 @@
   gem 'makara'
   gem 'activerecord-jdbcmysql-adapter', '>= 52', platform: :jruby
   gem 'activerecord-jdbcpostgresql-adapter', '>= 52', platform: :jruby
-  gem 'sequel', '~> 5.54.0' # TODO: Support sequel 5.62.0+
+  gem 'sequel'
   gem 'activerecord-jdbcsqlite3-adapter', '>= 52', platform: :jruby
 end
 

appraisal/jruby-9.3.rb

@@ -159,10 +159,15 @@
   gem 'http', '~> 4' # TODO: Fix test breakage and flakiness for 5+
   gem 'httpclient'
   gem 'rest-client'
-  gem 'stripe', '~> 8.0'
   gem 'typhoeus'
 end
 
+(7..12).each do |n|
+  appraise "stripe-#{n}" do
+    gem 'stripe', "~> #{n}"
+  end
+end
+
 [2, 3].each do |n|
   appraise "opensearch-#{n}" do
     gem 'opensearch-ruby', "~> #{n}"
@@ -182,7 +187,7 @@
   gem 'makara'
   gem 'activerecord-jdbcmysql-adapter', platform: :jruby
   gem 'activerecord-jdbcpostgresql-adapter', platform: :jruby
-  gem 'sequel', '~> 5.54.0' # TODO: Support sequel 5.62.0+
+  gem 'sequel'
   gem 'jdbc-sqlite3', '>= 3.28', platform: :jruby
 end