Timing out cfn-hup if it hangs (#2807)

Co-authored-by: Himani Deshpande <himanidp@amazon.com>
aws · Sep 16, 2024 · 13a71b4 · 13a71b4
1 parent 34b4b9d
commit 13a71b4
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb
@@ -98,5 +98,6 @@
   owner 'root'
   group 'root'
   mode '0744'
-  variables(cfn_bootstrap_virtualenv_path: virtualenv_path)
+  variables(cfn_bootstrap_virtualenv_path: virtualenv_path,
+            node_bootstrap_timeout: node['cluster']['compute_node_bootstrap_timeout'] || node['cluster']['Timeout'])
 end
diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb
@@ -8,12 +8,14 @@
       cached(:python_version) { '3.9.20' }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
+      cached(:timeout) { 1800 }
 
       context "when cfn_bootstrap virtualenv not installed yet" do
         cached(:chef_run) do
           runner = runner(platform: platform, version: version) do |node|
             node.override['cluster']['system_pyenv_root'] = system_pyenv_root
             node.override['cluster']['region'] = 'non_china'
+            node.override['cluster']['compute_node_bootstrap_timeout'] = timeout
           end
           runner.converge(described_recipe)
         end
@@ -79,7 +81,7 @@
             owner: 'root',
             group: 'root',
             mode: '0744',
-            variables: { cfn_bootstrap_virtualenv_path: virtualenv_path }
+            variables: { cfn_bootstrap_virtualenv_path: virtualenv_path, node_bootstrap_timeout: timeout }
           )
         end
       end

diff --git a/cookbooks/aws-parallelcluster-environment/templates/cfn_bootstrap/cfn-hup-runner.sh.erb b/cookbooks/aws-parallelcluster-environment/templates/cfn_bootstrap/cfn-hup-runner.sh.erb
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -x
 
 # This script is used to run cfn-hup is a way that is suitable to be managed by supervisord.
 # In particular, cfn-hup is executed in no-daemon mode.
@@ -9,6 +9,6 @@ set -ex
 [ -f /etc/profile.d/proxy.sh ] && . /etc/profile.d/proxy.sh
 
 while true; do
-  <%= @cfn_bootstrap_virtualenv_path %>/bin/cfn-hup --no-daemon --verbose
+  timeout <%= @node_bootstrap_timeout %> <%= @cfn_bootstrap_virtualenv_path %>/bin/cfn-hup --no-daemon --verbose
   sleep 60
 done