From 6d3fc1ab9ce15928f87367eecbbaef0b88da51ad Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Wed, 24 Jan 2024 11:07:30 +0100 Subject: [PATCH] split sigterm tests by web server for better balancing in parallel_tests --- test/spec/php-7.4_sigterm-apache2_spec.rb | 5 + ..._spec.rb => php-7.4_sigterm-nginx_spec.rb} | 2 +- test/spec/php-8.0_sigterm-apache2_spec.rb | 5 + ..._spec.rb => php-8.0_sigterm-nginx_spec.rb} | 2 +- test/spec/php-8.1_sigterm-apache2_spec.rb | 5 + ..._spec.rb => php-8.1_sigterm-nginx_spec.rb} | 2 +- test/spec/php-8.2_sigterm-apache2_spec.rb | 5 + ..._spec.rb => php-8.2_sigterm-nginx_spec.rb} | 2 +- test/spec/php-8.3_sigterm-apache2_spec.rb | 5 + ..._spec.rb => php-8.3_sigterm-nginx_spec.rb} | 2 +- test/spec/php_sigterm_shared.rb | 110 +++++++++--------- .../log/parallel_runtime_rspec.heroku-20.log | 15 ++- .../log/parallel_runtime_rspec.heroku-22.log | 9 +- 13 files changed, 99 insertions(+), 70 deletions(-) create mode 100644 test/spec/php-7.4_sigterm-apache2_spec.rb rename test/spec/{php-7.4_sigterm_spec.rb => php-7.4_sigterm-nginx_spec.rb} (92%) create mode 100644 test/spec/php-8.0_sigterm-apache2_spec.rb rename test/spec/{php-8.0_sigterm_spec.rb => php-8.0_sigterm-nginx_spec.rb} (92%) create mode 100644 test/spec/php-8.1_sigterm-apache2_spec.rb rename test/spec/{php-8.1_sigterm_spec.rb => php-8.1_sigterm-nginx_spec.rb} (92%) create mode 100644 test/spec/php-8.2_sigterm-apache2_spec.rb rename test/spec/{php-8.2_sigterm_spec.rb => php-8.2_sigterm-nginx_spec.rb} (92%) create mode 100644 test/spec/php-8.3_sigterm-apache2_spec.rb rename test/spec/{php-8.3_sigterm_spec.rb => php-8.3_sigterm-nginx_spec.rb} (92%) diff --git a/test/spec/php-7.4_sigterm-apache2_spec.rb b/test/spec/php-7.4_sigterm-apache2_spec.rb new file mode 100644 index 000000000..fc4f38f01 --- /dev/null +++ b/test/spec/php-7.4_sigterm-apache2_spec.rb @@ -0,0 +1,5 @@ +require_relative "php_sigterm_shared" + +describe "A PHP 7.4 application with long-running requests", :requires_php_on_stack => "7.4" do + include_examples "A PHP application with long-running requests", "7.4", "apache2" +end diff --git a/test/spec/php-7.4_sigterm_spec.rb b/test/spec/php-7.4_sigterm-nginx_spec.rb similarity index 92% rename from test/spec/php-7.4_sigterm_spec.rb rename to test/spec/php-7.4_sigterm-nginx_spec.rb index b09dbc96d..9bfebf4aa 100644 --- a/test/spec/php-7.4_sigterm_spec.rb +++ b/test/spec/php-7.4_sigterm-nginx_spec.rb @@ -1,5 +1,5 @@ require_relative "php_sigterm_shared" describe "A PHP 7.4 application with long-running requests", :requires_php_on_stack => "7.4" do - include_examples "A PHP application with long-running requests", "7.4" + include_examples "A PHP application with long-running requests", "7.4", "nginx" end diff --git a/test/spec/php-8.0_sigterm-apache2_spec.rb b/test/spec/php-8.0_sigterm-apache2_spec.rb new file mode 100644 index 000000000..0e18f9380 --- /dev/null +++ b/test/spec/php-8.0_sigterm-apache2_spec.rb @@ -0,0 +1,5 @@ +require_relative "php_sigterm_shared" + +describe "A PHP 8.0 application with long-running requests", :requires_php_on_stack => "8.0" do + include_examples "A PHP application with long-running requests", "8.0", "apache2" +end diff --git a/test/spec/php-8.0_sigterm_spec.rb b/test/spec/php-8.0_sigterm-nginx_spec.rb similarity index 92% rename from test/spec/php-8.0_sigterm_spec.rb rename to test/spec/php-8.0_sigterm-nginx_spec.rb index 5a3ac52aa..b605c8ef6 100644 --- a/test/spec/php-8.0_sigterm_spec.rb +++ b/test/spec/php-8.0_sigterm-nginx_spec.rb @@ -1,5 +1,5 @@ require_relative "php_sigterm_shared" describe "A PHP 8.0 application with long-running requests", :requires_php_on_stack => "8.0" do - include_examples "A PHP application with long-running requests", "8.0" + include_examples "A PHP application with long-running requests", "8.0", "nginx" end diff --git a/test/spec/php-8.1_sigterm-apache2_spec.rb b/test/spec/php-8.1_sigterm-apache2_spec.rb new file mode 100644 index 000000000..f40a03b50 --- /dev/null +++ b/test/spec/php-8.1_sigterm-apache2_spec.rb @@ -0,0 +1,5 @@ +require_relative "php_sigterm_shared" + +describe "A PHP 8.1 application with long-running requests", :requires_php_on_stack => "8.1" do + include_examples "A PHP application with long-running requests", "8.1", "apache2" +end diff --git a/test/spec/php-8.1_sigterm_spec.rb b/test/spec/php-8.1_sigterm-nginx_spec.rb similarity index 92% rename from test/spec/php-8.1_sigterm_spec.rb rename to test/spec/php-8.1_sigterm-nginx_spec.rb index 5d20272d2..9a141b5fd 100644 --- a/test/spec/php-8.1_sigterm_spec.rb +++ b/test/spec/php-8.1_sigterm-nginx_spec.rb @@ -1,5 +1,5 @@ require_relative "php_sigterm_shared" describe "A PHP 8.1 application with long-running requests", :requires_php_on_stack => "8.1" do - include_examples "A PHP application with long-running requests", "8.1" + include_examples "A PHP application with long-running requests", "8.1", "nginx" end diff --git a/test/spec/php-8.2_sigterm-apache2_spec.rb b/test/spec/php-8.2_sigterm-apache2_spec.rb new file mode 100644 index 000000000..ac532023b --- /dev/null +++ b/test/spec/php-8.2_sigterm-apache2_spec.rb @@ -0,0 +1,5 @@ +require_relative "php_sigterm_shared" + +describe "A PHP 8.2 application with long-running requests", :requires_php_on_stack => "8.2" do + include_examples "A PHP application with long-running requests", "8.2", "apache2" +end diff --git a/test/spec/php-8.2_sigterm_spec.rb b/test/spec/php-8.2_sigterm-nginx_spec.rb similarity index 92% rename from test/spec/php-8.2_sigterm_spec.rb rename to test/spec/php-8.2_sigterm-nginx_spec.rb index c9274771d..402730292 100644 --- a/test/spec/php-8.2_sigterm_spec.rb +++ b/test/spec/php-8.2_sigterm-nginx_spec.rb @@ -1,5 +1,5 @@ require_relative "php_sigterm_shared" describe "A PHP 8.2 application with long-running requests", :requires_php_on_stack => "8.2" do - include_examples "A PHP application with long-running requests", "8.2" + include_examples "A PHP application with long-running requests", "8.2", "nginx" end diff --git a/test/spec/php-8.3_sigterm-apache2_spec.rb b/test/spec/php-8.3_sigterm-apache2_spec.rb new file mode 100644 index 000000000..7b6e8f614 --- /dev/null +++ b/test/spec/php-8.3_sigterm-apache2_spec.rb @@ -0,0 +1,5 @@ +require_relative "php_sigterm_shared" + +describe "A PHP 8.3 application with long-running requests", :requires_php_on_stack => "8.3" do + include_examples "A PHP application with long-running requests", "8.3", "apache2" +end diff --git a/test/spec/php-8.3_sigterm_spec.rb b/test/spec/php-8.3_sigterm-nginx_spec.rb similarity index 92% rename from test/spec/php-8.3_sigterm_spec.rb rename to test/spec/php-8.3_sigterm-nginx_spec.rb index ed71c6854..9f436dd76 100644 --- a/test/spec/php-8.3_sigterm_spec.rb +++ b/test/spec/php-8.3_sigterm-nginx_spec.rb @@ -1,5 +1,5 @@ require_relative "php_sigterm_shared" describe "A PHP 8.3 application with long-running requests", :requires_php_on_stack => "8.3" do - include_examples "A PHP application with long-running requests", "8.3" + include_examples "A PHP application with long-running requests", "8.3", "nginx" end diff --git a/test/spec/php_sigterm_shared.rb b/test/spec/php_sigterm_shared.rb index 2a745b2d7..f74190dfa 100644 --- a/test/spec/php_sigterm_shared.rb +++ b/test/spec/php_sigterm_shared.rb @@ -1,7 +1,7 @@ require_relative "spec_helper" -shared_examples "A PHP application with long-running requests" do |series| - context "that uses #{series}" do +shared_examples "A PHP application with long-running requests" do |series, server| + context "that uses PHP #{series} and the #{server} web server" do before(:all) do @app = new_app_with_stack_and_platrepo('test/fixtures/sigterm', before_deploy: -> { system("composer require --quiet --ignore-platform-reqs php '#{series}.*'") or raise "Failed to require PHP version" }, @@ -14,61 +14,57 @@ @app.teardown! end - ['apache2', 'nginx'].each do |server| - context "running the #{server} web server" do - it "gracefully shuts down when the leader process receives a SIGTERM" do - # first, launch in the background and get the pid - # then sleep five seconds to allow boot (semicolon before ! needs a space, Bash...) - # curl the sleep() script, kill it after two seconds - # wait for $pid so that we can be certain to get all the output - cmd = "heroku-php-#{server} & pid=$! ; sleep 5; curl \"localhost:$PORT/index.php?wait=5\" & sleep 2; kill $pid; wait $pid" - retry_until retry: 3, sleep: 5 do - output = @app.run(cmd) - expect(output).to match(/^hello world after 5 second\(s\)$/) - expect(output).to match(/^request complete$/) # ensure a late log line is captured, meaning the logs tail process stays alive until the end - end - end - - it "gracefully shuts down when all processes receive a SIGTERM because HEROKU_PHP_GRACEFUL_SIGTERM is on by default" do - # first, launch in the background and get the pid - # then sleep five seconds to allow boot (semicolon before ! needs a space, Bash...) - # curl the sleep() script (and remember the curl pid) - # pgrep all our user's PIDs, then inverse-grep away $$ (that's our shell) and the curl PID - # hand all those PIDs to kill - # wait for $pid so that we can be certain to get all the output - cmd = "heroku-php-#{server} & pid=$! ; sleep 5; curl \"localhost:$PORT/index.php?wait=5\" & curlpid=$!; sleep 2; kill $(pgrep -U $UID | grep -vw -e $$ -e $curlpid) 2>/dev/null; wait $pid" - retry_until retry: 3, sleep: 5 do - output = @app.run(cmd) - expect(output).to match(/^hello world after 5 second\(s\)$/) - expect(output).to match(/^request complete$/) # ensure a late log line is captured, meaning the logs tail process stays alive until the end - end - end - - it "logs slowness, prints a trace, and terminates the process after configured timeouts" do - # launch web server wrapped in a 10 second timeout - # once web server is ready, `read` unblocks and we curl the sleep() script which will take a few seconds to run - # after `curl` completes, `waitforit.sh` will shut down - cmd = "./waitforit.sh 10 'ready for connections' heroku-php-#{server} -F fpm.request_slowlog_timeout.conf --verbose | { read && curl \"localhost:$PORT/index.php?wait=5\"; }" - retry_until retry: 3, sleep: 5 do - output = @app.run(cmd) - # ensure slowlog info and trace is there - expect(output).to include("executing too slow") - expect(output).to include("sleep() /app/index.php:5") - # ensure termination info is there - expect(output).to match(/execution timed out/) - end - end - - it "is configured to log slow requests after 3 seconds and terminate them after 30 seconds" do - # we can parse this from the config test output (-tt tests config and dumps PHP-FPM config) - cmd = "heroku-php-#{server} -tt" - retry_until retry: 3, sleep: 5 do - output = @app.run(cmd) - # ensure slowlog info and trace is there - expect(output).to include("request_slowlog_timeout = 3s") - expect(output).to include("request_terminate_timeout = 30s") - end - end + it "gracefully shuts down when the leader process receives a SIGTERM" do + # first, launch in the background and get the pid + # then sleep five seconds to allow boot (semicolon before ! needs a space, Bash...) + # curl the sleep() script, kill it after two seconds + # wait for $pid so that we can be certain to get all the output + cmd = "heroku-php-#{server} & pid=$! ; sleep 5; curl \"localhost:$PORT/index.php?wait=5\" & sleep 2; kill $pid; wait $pid" + retry_until retry: 3, sleep: 5 do + output = @app.run(cmd) + expect(output).to match(/^hello world after 5 second\(s\)$/) + expect(output).to match(/^request complete$/) # ensure a late log line is captured, meaning the logs tail process stays alive until the end + end + end + + it "gracefully shuts down when all processes receive a SIGTERM because HEROKU_PHP_GRACEFUL_SIGTERM is on by default" do + # first, launch in the background and get the pid + # then sleep five seconds to allow boot (semicolon before ! needs a space, Bash...) + # curl the sleep() script (and remember the curl pid) + # pgrep all our user's PIDs, then inverse-grep away $$ (that's our shell) and the curl PID + # hand all those PIDs to kill + # wait for $pid so that we can be certain to get all the output + cmd = "heroku-php-#{server} & pid=$! ; sleep 5; curl \"localhost:$PORT/index.php?wait=5\" & curlpid=$!; sleep 2; kill $(pgrep -U $UID | grep -vw -e $$ -e $curlpid) 2>/dev/null; wait $pid" + retry_until retry: 3, sleep: 5 do + output = @app.run(cmd) + expect(output).to match(/^hello world after 5 second\(s\)$/) + expect(output).to match(/^request complete$/) # ensure a late log line is captured, meaning the logs tail process stays alive until the end + end + end + + it "logs slowness, prints a trace, and terminates the process after configured timeouts" do + # launch web server wrapped in a 10 second timeout + # once web server is ready, `read` unblocks and we curl the sleep() script which will take a few seconds to run + # after `curl` completes, `waitforit.sh` will shut down + cmd = "./waitforit.sh 10 'ready for connections' heroku-php-#{server} -F fpm.request_slowlog_timeout.conf --verbose | { read && curl \"localhost:$PORT/index.php?wait=5\"; }" + retry_until retry: 3, sleep: 5 do + output = @app.run(cmd) + # ensure slowlog info and trace is there + expect(output).to include("executing too slow") + expect(output).to include("sleep() /app/index.php:5") + # ensure termination info is there + expect(output).to match(/execution timed out/) + end + end + + it "is configured to log slow requests after 3 seconds and terminate them after 30 seconds" do + # we can parse this from the config test output (-tt tests config and dumps PHP-FPM config) + cmd = "heroku-php-#{server} -tt" + retry_until retry: 3, sleep: 5 do + output = @app.run(cmd) + # ensure slowlog info and trace is there + expect(output).to include("request_slowlog_timeout = 3s") + expect(output).to include("request_terminate_timeout = 30s") end end end diff --git a/test/var/log/parallel_runtime_rspec.heroku-20.log b/test/var/log/parallel_runtime_rspec.heroku-20.log index 0b0fe2337..695edf3dc 100644 --- a/test/var/log/parallel_runtime_rspec.heroku-20.log +++ b/test/var/log/parallel_runtime_rspec.heroku-20.log @@ -16,30 +16,35 @@ test/spec/php-7.4_boot-apache2_spec.rb:110 test/spec/php-7.4_boot-nginx_spec.rb:100 test/spec/php-7.4_concurrency-apache2_spec.rb:100 test/spec/php-7.4_concurrency-nginx_spec.rb:90 -test/spec/php-7.4_sigterm_spec.rb:95 +test/spec/php-7.4_sigterm-apache2_spec.rb:50 +test/spec/php-7.4_sigterm-nginx_spec.rb:50 test/spec/php-8.0_base_spec.rb:52 test/spec/php-8.0_boot-apache2_spec.rb:110 test/spec/php-8.0_boot-nginx_spec.rb:100 test/spec/php-8.0_concurrency-apache2_spec.rb:100 test/spec/php-8.0_concurrency-nginx_spec.rb:90 -test/spec/php-8.0_sigterm_spec.rb:95 +test/spec/php-8.0_sigterm-apache2_spec.rb:50 +test/spec/php-8.0_sigterm-nginx_spec.rb:50 test/spec/php-8.1_base_spec.rb:52 test/spec/php-8.1_boot-apache2_spec.rb:110 test/spec/php-8.1_boot-nginx_spec.rb:100 test/spec/php-8.1_concurrency-apache2_spec.rb:100 test/spec/php-8.1_concurrency-nginx_spec.rb:90 -test/spec/php-8.1_sigterm_spec.rb:95 +test/spec/php-8.1_sigterm-apache2_spec.rb:50 +test/spec/php-8.1_sigterm-nginx_spec.rb:50 test/spec/php-8.2_base_spec.rb:52 test/spec/php-8.2_boot-apache2_spec.rb:110 test/spec/php-8.2_boot-nginx_spec.rb:100 test/spec/php-8.2_concurrency-apache2_spec.rb:100 test/spec/php-8.2_concurrency-nginx_spec.rb:90 -test/spec/php-8.2_sigterm_spec.rb:95 +test/spec/php-8.2_sigterm-apache2_spec.rb:50 +test/spec/php-8.2_sigterm-nginx_spec.rb:50 test/spec/php-8.3_base_spec.rb:52 test/spec/php-8.3_boot-apache2_spec.rb:110 test/spec/php-8.3_boot-nginx_spec.rb:100 test/spec/php-8.3_concurrency-apache2_spec.rb:100 test/spec/php-8.3_concurrency-nginx_spec.rb:90 -test/spec/php-8.3_sigterm_spec.rb:95 +test/spec/php-8.3_sigterm-apache2_spec.rb:50 +test/spec/php-8.3_sigterm-nginx_spec.rb:50 test/spec/php_default_spec.rb:80 test/spec/platform_spec.rb:65 diff --git a/test/var/log/parallel_runtime_rspec.heroku-22.log b/test/var/log/parallel_runtime_rspec.heroku-22.log index 7b6176f1b..1b0cac20f 100644 --- a/test/var/log/parallel_runtime_rspec.heroku-22.log +++ b/test/var/log/parallel_runtime_rspec.heroku-22.log @@ -11,18 +11,21 @@ test/spec/php-8.1_boot-apache2_spec.rb:110 test/spec/php-8.1_boot-nginx_spec.rb:100 test/spec/php-8.1_concurrency-apache2_spec.rb:100 test/spec/php-8.1_concurrency-nginx_spec.rb:90 -test/spec/php-8.1_sigterm_spec.rb:95 +test/spec/php-8.1_sigterm-apache2_spec.rb:50 +test/spec/php-8.1_sigterm-nginx_spec.rb:50 test/spec/php-8.2_base_spec.rb:52 test/spec/php-8.2_boot-apache2_spec.rb:110 test/spec/php-8.2_boot-nginx_spec.rb:100 test/spec/php-8.2_concurrency-apache2_spec.rb:100 test/spec/php-8.2_concurrency-nginx_spec.rb:90 -test/spec/php-8.2_sigterm_spec.rb:95 +test/spec/php-8.2_sigterm-apache2_spec.rb:50 +test/spec/php-8.2_sigterm-nginx_spec.rb:50 test/spec/php-8.3_base_spec.rb:52 test/spec/php-8.3_boot-apache2_spec.rb:110 test/spec/php-8.3_boot-nginx_spec.rb:100 test/spec/php-8.3_concurrency-apache2_spec.rb:100 test/spec/php-8.3_concurrency-nginx_spec.rb:90 -test/spec/php-8.3_sigterm_spec.rb:95 +test/spec/php-8.3_sigterm-apache2_spec.rb:50 +test/spec/php-8.3_sigterm-nginx_spec.rb:50 test/spec/php_default_spec.rb:80 test/spec/platform_spec.rb:65