reorganize benchmark validators again

benchmarks/README.md

@@ -2,6 +2,24 @@
 
 ## Adding a New Benchmark File
 
-- Add the new file to `run_all.sh` in this directory
-- Add the new file to `spec/validate_benchmarks_spec.rb`
-which validates that the benchmarks are error-free in the test suite.
+1. Use one of the following prefixes:
+
+  - `library_`
+  - `profiling_`
+  - `tracing_`
+
+2. Add the new file to `run_all.sh` in this directory.
+
+3. Depending on the prefix, add the new file to the correct
+  `validate_benchmarks_spec.rb` as follows:
+
+  - `library_` prefix: `spec/validate_benchmarks_spec.rb`
+  - `profiling_` prefix: `./spec/datadog/profiling/validate_benchmarks_spec.rb`
+  - `tracing_` prefix: `./spec/datadog/tracing/validate_benchmarks_spec.rb`
+
+## Adding Benchmarks For a New Product
+
+1. Create a `validate_benchmarks_spec.rb` test in the product subdirectory,
+  using the existing files as a template.
+
+2. Update this README to add the new product in the previous section.

benchmarks/run_all.sh

@@ -6,7 +6,7 @@
 set -ex
 
 for file in \
-  `dirname "$0"`/gem_loading.rb \
+  `dirname "$0"`/library_gem_loading.rb \
   `dirname "$0"`/profiler_allocation.rb \
   `dirname "$0"`/profiler_gc.rb \
   `dirname "$0"`/profiler_hold_resume_interruptions.rb \

spec/datadog/profiling/validate_benchmarks_spec.rb

@@ -10,13 +10,13 @@
   end
 
   benchmarks_to_validate = [
-    'profiler_sample_loop_v2',
-    'profiler_http_transport',
-    'profiler_sample_serialize',
-    'profiler_memory_sample_serialize',
+    'profiler_allocation',
     'profiler_gc',
     'profiler_hold_resume_interruptions',
-    'profiler_allocation',
+    'profiler_http_transport',
+    'profiler_memory_sample_serialize',
+    'profiler_sample_loop_v2',
+    'profiler_sample_serialize',
   ].freeze
 
   benchmarks_to_validate.each do |benchmark|

spec/datadog/tracing/validate_benchmarks_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+RSpec.describe 'Tracing benchmarks' do
+  before { skip('Spec requires Ruby VM supporting fork') unless PlatformHelpers.supports_fork? }
+
+  around do |example|
+    ClimateControl.modify('VALIDATE_BENCHMARK' => 'true') do
+      example.run
+    end
+  end
+
+  benchmarks_to_validate = %w(
+    tracing_trace
+  )
+
+  benchmarks_to_validate.each do |benchmark|
+    describe benchmark do
+      it 'runs without raising errors' do
+        expect_in_fork do
+          load "./benchmarks/#{benchmark}.rb"
+        end
+      end
+    end
+  end
+
+  # This test validates that we don't forget to add new benchmarks to benchmarks_to_validate
+  it 'tests all expected benchmarks in the benchmarks folder' do
+    all_benchmarks = Dir['./benchmarks/tracing_*'].map { |it| it.gsub('./benchmarks/', '').gsub('.rb', '') }
+
+    expect(benchmarks_to_validate).to contain_exactly(*all_benchmarks)
+  end
+end

spec/validate_benchmarks_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe 'Tracing benchmarks' do
+RSpec.describe 'Library benchmarks' do
   before { skip('Spec requires Ruby VM supporting fork') unless PlatformHelpers.supports_fork? }
 
   around do |example|
@@ -9,17 +9,11 @@
     end
   end
 
-  %w(
-    gem_loading
-    profiler_allocation
-    profiler_gc
-    profiler_hold_resume_interruptions
-    profiler_http_transport
-    profiler_memory_sample_serialize
-    profiler_sample_loop_v2
-    profiler_sample_serialize
-    tracing_trace
-  ).each do |benchmark|
+  benchmarks_to_validate = %w(
+    library_gem_loading
+  )
+
+  benchmarks_to_validate.each do |benchmark|
     describe benchmark do
       it 'runs without raising errors' do
         expect_in_fork do
@@ -28,4 +22,11 @@
       end
     end
   end
+
+  # This test validates that we don't forget to add new benchmarks to benchmarks_to_validate
+  it 'tests all expected benchmarks in the benchmarks folder' do
+    all_benchmarks = Dir['./benchmarks/library_*'].map { |it| it.gsub('./benchmarks/', '').gsub('.rb', '') }
+
+    expect(benchmarks_to_validate).to contain_exactly(*all_benchmarks)
+  end
 end