feat: custom lifecyclehooks for machinepools (#613)

Backported from kubernetes-sigs#4875 Co-authored-by: Sebastien Michel <semichel@cisco.com>
giantswarm · Dec 2, 2024 · 09d9674 · 09d9674
1 parent bf3472a
commit 09d9674
Show file tree

Hide file tree

Showing 39 changed files with 1,187 additions and 92 deletions.
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go
@@ -177,6 +177,9 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"elasticloadbalancing:DeleteListener",
 				"autoscaling:DescribeAutoScalingGroups",
 				"autoscaling:DescribeInstanceRefreshes",
+				"autoscaling:DeleteLifecycleHook",
+				"autoscaling:DescribeLifecycleHooks",
+				"autoscaling:PutLifecycleHook",
 				"ec2:CreateLaunchTemplate",
 				"ec2:CreateLaunchTemplateVersion",
 				"ec2:DescribeLaunchTemplates",

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml
@@ -243,6 +243,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml
@@ -243,6 +243,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml
@@ -243,6 +243,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml
@@ -243,6 +243,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml
@@ -237,6 +237,9 @@ Resources:
           - elasticloadbalancing:DeleteListener
           - autoscaling:DescribeAutoScalingGroups
           - autoscaling:DescribeInstanceRefreshes
+          - autoscaling:DeleteLifecycleHook
+          - autoscaling:DescribeLifecycleHooks
+          - autoscaling:PutLifecycleHook
           - ec2:CreateLaunchTemplate
           - ec2:CreateLaunchTemplateVersion
           - ec2:DescribeLaunchTemplates

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml
@@ -983,6 +983,55 @@ spec:
                     - "3.4"
                     type: string
                 type: object
+              lifecycleHooks:
+                description: AWSLifecycleHooks specifies lifecycle hooks for the autoscaling
+                  group.
+                items:
+                  description: AWSLifecycleHook describes an AWS lifecycle hook
+                  properties:
+                    defaultResult:
+                      description: The default result for the lifecycle hook. The
+                        possible values are CONTINUE and ABANDON.
+                      enum:
+                      - CONTINUE
+                      - ABANDON
+                      type: string
+                    heartbeatTimeout:
+                      description: |-
+                        The maximum time, in seconds, that an instance can remain in a Pending:Wait or
+                        Terminating:Wait state. The maximum is 172800 seconds (48 hours) or 100 times
+                        HeartbeatTimeout, whichever is smaller.
+                      format: duration
+                      type: string
+                    lifecycleTransition:
+                      description: The state of the EC2 instance to which to attach
+                        the lifecycle hook.
+                      enum:
+                      - autoscaling:EC2_INSTANCE_LAUNCHING
+                      - autoscaling:EC2_INSTANCE_TERMINATING
+                      type: string
+                    name:
+                      description: The name of the lifecycle hook.
+                      type: string
+                    notificationMetadata:
+                      description: Contains additional metadata that will be passed
+                        to the notification target.
+                      type: string
+                    notificationTargetARN:
+                      description: |-
+                        The ARN of the notification target that Amazon EC2 Auto Scaling uses to
+                        notify you when an instance is in the transition state for the lifecycle hook.
+                      type: string
+                    roleARN:
+                      description: |-
+                        The ARN of the IAM role that allows the Auto Scaling group to publish to the
+                        specified notification target.
+                      type: string
+                  required:
+                  - lifecycleTransition
+                  - name
+                  type: object
+                type: array
               maxSize:
                 default: 1
                 description: MaxSize defines the maximum size of the group.

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml
@@ -890,6 +890,55 @@ spec:
                   type: string
                 description: Labels specifies labels for the Kubernetes node objects
                 type: object
+              lifecycleHooks:
+                description: AWSLifecycleHooks specifies lifecycle hooks for the managed
+                  node group.
+                items:
+                  description: AWSLifecycleHook describes an AWS lifecycle hook
+                  properties:
+                    defaultResult:
+                      description: The default result for the lifecycle hook. The
+                        possible values are CONTINUE and ABANDON.
+                      enum:
+                      - CONTINUE
+                      - ABANDON
+                      type: string
+                    heartbeatTimeout:
+                      description: |-
+                        The maximum time, in seconds, that an instance can remain in a Pending:Wait or
+                        Terminating:Wait state. The maximum is 172800 seconds (48 hours) or 100 times
+                        HeartbeatTimeout, whichever is smaller.
+                      format: duration
+                      type: string
+                    lifecycleTransition:
+                      description: The state of the EC2 instance to which to attach
+                        the lifecycle hook.
+                      enum:
+                      - autoscaling:EC2_INSTANCE_LAUNCHING
+                      - autoscaling:EC2_INSTANCE_TERMINATING
+                      type: string
+                    name:
+                      description: The name of the lifecycle hook.
+                      type: string
+                    notificationMetadata:
+                      description: Contains additional metadata that will be passed
+                        to the notification target.
+                      type: string
+                    notificationTargetARN:
+                      description: |-
+                        The ARN of the notification target that Amazon EC2 Auto Scaling uses to
+                        notify you when an instance is in the transition state for the lifecycle hook.
+                      type: string
+                    roleARN:
+                      description: |-
+                        The ARN of the IAM role that allows the Auto Scaling group to publish to the
+                        specified notification target.
+                      type: string
+                  required:
+                  - lifecycleTransition
+                  - name
+                  type: object
+                type: array
               providerIDList:
                 description: |-
                   ProviderIDList are the provider IDs of instances in the

diff --git a/exp/api/v1beta1/conversion.go b/exp/api/v1beta1/conversion.go
@@ -55,6 +55,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Spec.Ignition != nil {
 		dst.Spec.Ignition = restored.Spec.Ignition
 	}
+	if restored.Spec.AWSLifecycleHooks != nil {
+		dst.Spec.AWSLifecycleHooks = restored.Spec.AWSLifecycleHooks
+	}
 
 	if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
 		dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName
@@ -116,6 +119,9 @@ func (src *AWSManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Spec.AvailabilityZoneSubnetType != nil {
 		dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
 	}
+	if restored.Spec.AWSLifecycleHooks != nil {
+		dst.Spec.AWSLifecycleHooks = restored.Spec.AWSLifecycleHooks
+	}
 
 	return nil
 }

diff --git a/exp/api/v1beta1/zz_generated.conversion.go b/exp/api/v1beta1/zz_generated.conversion.go
diff --git a/exp/api/v1beta2/awsmachinepool_types.go b/exp/api/v1beta2/awsmachinepool_types.go
@@ -105,6 +105,10 @@ type AWSMachinePoolSpec struct {
 	// Ignition defined options related to the bootstrapping systems where Ignition is used.
 	// +optional
 	Ignition *infrav1.Ignition `json:"ignition,omitempty"`
+
+	// AWSLifecycleHooks specifies lifecycle hooks for the autoscaling group.
+	// +optional
+	AWSLifecycleHooks []AWSLifecycleHook `json:"lifecycleHooks,omitempty"`
 }
 
 // SuspendProcessesTypes contains user friendly auto-completable values for suspended process names.

diff --git a/exp/api/v1beta2/awsmachinepool_webhook.go b/exp/api/v1beta2/awsmachinepool_webhook.go
@@ -179,6 +179,10 @@ func (r *AWSMachinePool) validateIgnition() field.ErrorList {
 	return allErrs
 }
 
+func (r *AWSMachinePool) validateLifecycleHooks() field.ErrorList {
+	return validateLifecycleHooks(r.Spec.AWSLifecycleHooks)
+}
+
 // ValidateCreate will do any extra validation when creating a AWSMachinePool.
 func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
@@ -194,6 +198,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
 	allErrs = append(allErrs, r.validateSpotInstances()...)
 	allErrs = append(allErrs, r.validateRefreshPreferences()...)
 	allErrs = append(allErrs, r.validateIgnition()...)
+	allErrs = append(allErrs, r.validateLifecycleHooks()...)
 
 	if len(allErrs) == 0 {
 		return nil, nil
@@ -216,6 +221,7 @@ func (r *AWSMachinePool) ValidateUpdate(_ runtime.Object) (admission.Warnings, e
 	allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
 	allErrs = append(allErrs, r.validateSpotInstances()...)
 	allErrs = append(allErrs, r.validateRefreshPreferences()...)
+	allErrs = append(allErrs, r.validateLifecycleHooks()...)
 
 	if len(allErrs) == 0 {
 		return nil, nil