diff --git a/api/v1beta2/conditions_consts.go b/api/v1beta2/conditions_consts.go index c76dd13d5b..9cd1870a99 100644 --- a/api/v1beta2/conditions_consts.go +++ b/api/v1beta2/conditions_consts.go @@ -87,6 +87,14 @@ const ( RouteTableReconciliationFailedReason = "RouteTableReconciliationFailed" ) +const ( + // VpcEndpointsReadyCondition reports successful reconciliation of vpc endpoints. + // Only applicable to managed clusters. + VpcEndpointsReadyCondition clusterv1.ConditionType = "VpcEndpointsReadyCondition" + // VpcEndpointsReconciliationFailedReason used when any errors occur during reconciliation of vpc endpoints. + VpcEndpointsReconciliationFailedReason = "VpcEndpointsReconciliationFailed" +) + const ( // SecondaryCidrsReadyCondition reports successful reconciliation of secondary CIDR blocks. // Only applicable to managed clusters. diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go index 616e60d164..3b44b5ec2a 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go @@ -102,7 +102,9 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { "ec2:CreateSubnet", "ec2:CreateTags", "ec2:CreateVpc", + "ec2:CreateVpcEndpoint", "ec2:ModifyVpcAttribute", + "ec2:ModifyVpcEndpoint", "ec2:DeleteInternetGateway", "ec2:DeleteEgressOnlyInternetGateway", "ec2:DeleteNatGateway", @@ -112,6 +114,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DeleteVpc", + "ec2:DeleteVpcEndpoints", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeAvailabilityZones", @@ -129,6 +132,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { "ec2:DescribeSubnets", "ec2:DescribeVpcs", "ec2:DescribeVpcAttribute", + "ec2:DescribeVpcEndpoints", "ec2:DescribeVolumes", "ec2:DescribeTags", "ec2:DetachInternetGateway", diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml index 74a6a8ebb1..d5b5505009 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml index 74f0a3c0c6..a2c06a7e28 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml index 65dc75ed70..e965556dc5 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml @@ -167,7 +167,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -177,6 +179,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -194,6 +197,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml index 4a3080ac2c..46b4121509 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml index c28eee7bc7..6fc278b78a 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml @@ -167,7 +167,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -177,6 +179,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -194,6 +197,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml index 0c87cb7a28..4cb1a565cf 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml @@ -167,7 +167,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -177,6 +179,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -194,6 +197,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml index 24e9168542..1dd528076b 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml index 901e52da36..80f96c8d6d 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml index 99ae19e1bc..9ce26aff22 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml index 1e71d0dbac..76af2c7aee 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml index da8b66ef69..67e78b9504 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml index 3455ca7c4d..ef5fd59980 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml @@ -167,7 +167,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -177,6 +179,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -194,6 +197,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml index a045a10bf0..9344f79aa0 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml index 9a3c810b38..472fdaacf3 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml @@ -161,7 +161,9 @@ Resources: - ec2:CreateSubnet - ec2:CreateTags - ec2:CreateVpc + - ec2:CreateVpcEndpoint - ec2:ModifyVpcAttribute + - ec2:ModifyVpcEndpoint - ec2:DeleteInternetGateway - ec2:DeleteEgressOnlyInternetGateway - ec2:DeleteNatGateway @@ -171,6 +173,7 @@ Resources: - ec2:DeleteSubnet - ec2:DeleteTags - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeAvailabilityZones @@ -188,6 +191,7 @@ Resources: - ec2:DescribeSubnets - ec2:DescribeVpcs - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints - ec2:DescribeVolumes - ec2:DescribeTags - ec2:DetachInternetGateway diff --git a/controllers/awscluster_controller_test.go b/controllers/awscluster_controller_test.go index 28209c9412..1ab83dfd8a 100644 --- a/controllers/awscluster_controller_test.go +++ b/controllers/awscluster_controller_test.go @@ -263,7 +263,7 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) { }) }) - t.Run("Should fail on AWSCluster reconciliation if VPC limit exceeded", func(t *testing.T) { + t.Run("Should fail on AWSCluster reconciliation if `VPC limit exceeded`", func(t *testing.T) { // Assuming the max VPC limit is 2 and when two VPCs are created, the creation of 3rd VPC throws mocked error from EC2 API g := NewWithT(t) mockCtrl = gomock.NewController(t) @@ -416,6 +416,7 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) { {infrav1.BastionHostReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, {infrav1.SecondaryCidrsReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletingReason}, {infrav1.RouteTablesReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, + {infrav1.VpcEndpointsReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, {infrav1.NatGatewaysReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, {infrav1.InternetGatewayReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, {infrav1.SubnetsReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}, @@ -710,6 +711,24 @@ func mockedDeleteVPCCallsForNonExistentVPC(m *mocks.MockEC2APIMockRecorder) { } func mockedDeleteVPCCalls(m *mocks.MockEC2APIMockRecorder) { + m.DescribeVpcEndpointsPages(gomock.Eq(&ec2.DescribeVpcEndpointsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: aws.StringSlice([]string{"vpc-exists"}), + }, + }}), + gomock.Any()).Do(func(_, y interface{}) { + funct := y.(func(page *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool) + funct(&ec2.DescribeVpcEndpointsOutput{VpcEndpoints: []*ec2.VpcEndpoint{{ + VpcEndpointId: aws.String("vpce-12345"), + }}}, true) + }).Return(nil).AnyTimes() + + m.DeleteVpcEndpoints(gomock.Eq(&ec2.DeleteVpcEndpointsInput{ + VpcEndpointIds: aws.StringSlice([]string{"vpce-12345"}), + })).Return(&ec2.DeleteVpcEndpointsOutput{}, nil).AnyTimes() + m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ Filters: []*ec2.Filter{ { diff --git a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go index 938f31dadf..2a8ff4f9c0 100644 --- a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go +++ b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go @@ -204,6 +204,7 @@ func (r *AWSManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ct infrav1.InternetGatewayReadyCondition, infrav1.NatGatewaysReadyCondition, infrav1.RouteTablesReadyCondition, + infrav1.VpcEndpointsReadyCondition, ) if managedScope.Bastion().Enabled { applicableConditions = append(applicableConditions, infrav1.BastionHostReadyCondition) diff --git a/pkg/cloud/scope/cluster.go b/pkg/cloud/scope/cluster.go index 1f939fdd0f..592bd8c100 100644 --- a/pkg/cloud/scope/cluster.go +++ b/pkg/cloud/scope/cluster.go @@ -234,7 +234,9 @@ func (s *ClusterScope) PatchObject() error { applicableConditions = append(applicableConditions, infrav1.InternetGatewayReadyCondition, infrav1.NatGatewaysReadyCondition, - infrav1.RouteTablesReadyCondition) + infrav1.RouteTablesReadyCondition, + infrav1.VpcEndpointsReadyCondition, + ) if s.AWSCluster.Spec.Bastion.Enabled { applicableConditions = append(applicableConditions, infrav1.BastionHostReadyCondition) @@ -261,6 +263,7 @@ func (s *ClusterScope) PatchObject() error { infrav1.EgressOnlyInternetGatewayReadyCondition, infrav1.NatGatewaysReadyCondition, infrav1.RouteTablesReadyCondition, + infrav1.VpcEndpointsReadyCondition, infrav1.ClusterSecurityGroupsReadyCondition, infrav1.BastionHostReadyCondition, infrav1.LoadBalancerReadyCondition, diff --git a/pkg/cloud/scope/managedcontrolplane.go b/pkg/cloud/scope/managedcontrolplane.go index 63b5f895d6..56e5bd59c5 100644 --- a/pkg/cloud/scope/managedcontrolplane.go +++ b/pkg/cloud/scope/managedcontrolplane.go @@ -251,6 +251,7 @@ func (s *ManagedControlPlaneScope) PatchObject() error { infrav1.InternetGatewayReadyCondition, infrav1.NatGatewaysReadyCondition, infrav1.RouteTablesReadyCondition, + infrav1.VpcEndpointsReadyCondition, infrav1.BastionHostReadyCondition, infrav1.EgressOnlyInternetGatewayReadyCondition, ekscontrolplanev1.EKSControlPlaneCreatingCondition, @@ -307,6 +308,12 @@ func (s *ManagedControlPlaneScope) Bastion() *infrav1.Bastion { return &s.ControlPlane.Spec.Bastion } +// Bucket returns the bucket details. +// For ManagedControlPlane this is always nil, as we don't support S3 buckets for managed clusters. +func (s *ManagedControlPlaneScope) Bucket() *infrav1.S3Bucket { + return nil +} + // TagUnmanagedNetworkResources returns if the feature flag tag unmanaged network resources is set. func (s *ManagedControlPlaneScope) TagUnmanagedNetworkResources() bool { return s.tagUnmanagedNetworkResources diff --git a/pkg/cloud/scope/network.go b/pkg/cloud/scope/network.go index b35bc3d3ce..32b02ca0d2 100644 --- a/pkg/cloud/scope/network.go +++ b/pkg/cloud/scope/network.go @@ -43,6 +43,9 @@ type NetworkScope interface { // Bastion returns the bastion details for the cluster. Bastion() *infrav1.Bastion + // Bucket returns the cluster bucket. + Bucket() *infrav1.S3Bucket + // TagUnmanagedNetworkResources returns is tagging unmanaged network resources is set. TagUnmanagedNetworkResources() bool diff --git a/pkg/cloud/services/network/network.go b/pkg/cloud/services/network/network.go index 6e0572f7a9..b2363b5aac 100644 --- a/pkg/cloud/services/network/network.go +++ b/pkg/cloud/services/network/network.go @@ -73,6 +73,12 @@ func (s *Service) ReconcileNetwork() (err error) { return err } + // VPC Endpoints. + if err := s.reconcileVPCEndpoints(); err != nil { + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.VpcEndpointsReadyCondition, infrav1.VpcEndpointsReconciliationFailedReason, infrautilconditions.ErrorConditionAfterInit(s.scope.ClusterObj()), err.Error()) + return err + } + s.scope.Debug("Reconcile network completed successfully") return nil } @@ -99,6 +105,18 @@ func (s *Service) DeleteNetwork() (err error) { vpc.DeepCopyInto(s.scope.VPC()) + // VPC Endpoints. + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.VpcEndpointsReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") + if err := s.scope.PatchObject(); err != nil { + return err + } + + if err := s.deleteVPCEndpoints(); err != nil { + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.VpcEndpointsReadyCondition, "DeletingFailed", clusterv1.ConditionSeverityWarning, err.Error()) + return err + } + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.VpcEndpointsReadyCondition, clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") + // Routing tables. conditions.MarkFalse(s.scope.InfraCluster(), infrav1.RouteTablesReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") if err := s.scope.PatchObject(); err != nil { diff --git a/pkg/cloud/services/network/vpc.go b/pkg/cloud/services/network/vpc.go index 05d19373c2..94face8454 100644 --- a/pkg/cloud/services/network/vpc.go +++ b/pkg/cloud/services/network/vpc.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors" @@ -133,6 +134,155 @@ func (s *Service) reconcileVPC() error { return nil } +func (s *Service) describeVPCEndpoints(filters ...*ec2.Filter) ([]*ec2.VpcEndpoint, error) { + vpc := s.scope.VPC() + if vpc == nil || vpc.ID == "" { + return nil, errors.New("vpc is nil or vpc id is not set") + } + input := &ec2.DescribeVpcEndpointsInput{ + Filters: append(filters, &ec2.Filter{ + Name: aws.String("vpc-id"), + Values: []*string{&vpc.ID}, + }), + } + endpoints := []*ec2.VpcEndpoint{} + if err := s.EC2Client.DescribeVpcEndpointsPages(input, func(dveo *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool { + endpoints = append(endpoints, dveo.VpcEndpoints...) + return true + }); err != nil { + return nil, errors.Wrap(err, "failed to describe vpc endpoints") + } + return endpoints, nil +} + +// reconcileVPCEndpoints registers the AWS endpoints for the services that need to be enabled +// in the VPC routing tables. If the VPC is unmanaged, this is a no-op. +// For more information, see: https://docs.aws.amazon.com/vpc/latest/privatelink/gateway-endpoints.html +func (s *Service) reconcileVPCEndpoints() error { + // If the VPC is unmanaged or not yet populated, return early. + if s.scope.VPC().IsUnmanaged(s.scope.Name()) || s.scope.VPC().ID == "" { + return nil + } + + // Gather all services that need to be enabled. + services := sets.New[string]() + if s.scope.Bucket() != nil { + services.Insert(fmt.Sprintf("com.amazonaws.%s.s3", s.scope.Region())) + } + if services.Len() == 0 { + return nil + } + + // Gather the current routes. + routeTables := sets.New[string]() + for _, rt := range s.scope.Subnets() { + if rt.RouteTableID != nil && *rt.RouteTableID != "" { + routeTables.Insert(*rt.RouteTableID) + } + } + if routeTables.Len() == 0 { + return nil + } + + // Build the filters based on all the services we need to enable. + // A single filter with multiple values functions as an OR. + filters := []*ec2.Filter{ + { + Name: aws.String("service-name"), + Values: aws.StringSlice(services.UnsortedList()), + }, + } + + // Get all existing endpoints. + endpoints, err := s.describeVPCEndpoints(filters...) + if err != nil { + return errors.Wrap(err, "failed to describe vpc endpoints") + } + + // Iterate over all services and create missing endpoints. + for _, service := range services.UnsortedList() { + var existing *ec2.VpcEndpoint + for _, ep := range endpoints { + if aws.StringValue(ep.ServiceName) == service { + existing = ep + break + } + } + + // Handle the case where the endpoint already exists. + // If the route tables are different, modify the endpoint. + if existing != nil { + existingRouteTables := sets.New(aws.StringValueSlice(existing.RouteTableIds)...) + existingRouteTables.Delete("") + additions := routeTables.Difference(existingRouteTables) + removals := existingRouteTables.Difference(routeTables) + if additions.Len() > 0 || removals.Len() > 0 { + modify := &ec2.ModifyVpcEndpointInput{ + VpcEndpointId: existing.VpcEndpointId, + } + if additions.Len() > 0 { + modify.AddRouteTableIds = aws.StringSlice(additions.UnsortedList()) + } + if removals.Len() > 0 { + modify.RemoveRouteTableIds = aws.StringSlice(removals.UnsortedList()) + } + if _, err := s.EC2Client.ModifyVpcEndpoint(modify); err != nil { + return errors.Wrapf(err, "failed to modify vpc endpoint for service %q", service) + } + } + continue + } + + // Create the endpoint. + if _, err := s.EC2Client.CreateVpcEndpoint(&ec2.CreateVpcEndpointInput{ + VpcId: aws.String(s.scope.VPC().ID), + ServiceName: aws.String(service), + RouteTableIds: aws.StringSlice(routeTables.UnsortedList()), + TagSpecifications: []*ec2.TagSpecification{ + tags.BuildParamsToTagSpecification(ec2.ResourceTypeVpcEndpoint, s.getVPCEndpointTagParams()), + }, + }); err != nil { + return errors.Wrapf(err, "failed to create vpc endpoint for service %q", service) + } + } + + return nil +} + +func (s *Service) deleteVPCEndpoints() error { + // If the VPC is unmanaged or not yet populated, return early. + if s.scope.VPC().IsUnmanaged(s.scope.Name()) || s.scope.VPC().ID == "" { + return nil + } + + // Get all existing endpoints. + endpoints, err := s.describeVPCEndpoints() + if err != nil { + return errors.Wrap(err, "failed to describe vpc endpoints") + } + + // Gather all endpoint IDs. + ids := []*string{} + for _, ep := range endpoints { + if ep.VpcEndpointId == nil || *ep.VpcEndpointId == "" { + continue + } + ids = append(ids, ep.VpcEndpointId) + } + + if len(ids) == 0 { + return nil + } + + // Iterate over all services and delete endpoints. + if _, err := s.EC2Client.DeleteVpcEndpoints(&ec2.DeleteVpcEndpointsInput{ + VpcEndpointIds: ids, + }); err != nil { + return errors.Wrapf(err, "failed to delete vpc endpoints %+v", ids) + } + return nil +} + func (s *Service) ensureManagedVPCAttributes(vpc *infrav1.VPCSpec) error { var ( errs []error @@ -435,3 +585,12 @@ func (s *Service) getVPCTagParams(id string) infrav1.BuildParams { Additional: s.scope.AdditionalTags(), } } + +func (s *Service) getVPCEndpointTagParams() infrav1.BuildParams { + return infrav1.BuildParams{ + ClusterName: s.scope.Name(), + Lifecycle: infrav1.ResourceLifecycleOwned, + Role: aws.String(infrav1.CommonRoleTagValue), + Additional: s.scope.AdditionalTags(), + } +} diff --git a/test/e2e/shared/aws.go b/test/e2e/shared/aws.go index 8e39d83d67..c7f6be8fee 100644 --- a/test/e2e/shared/aws.go +++ b/test/e2e/shared/aws.go @@ -1207,6 +1207,29 @@ func GetVPCByName(e2eCtx *E2EContext, vpcName string) (*ec2.Vpc, error) { return result.Vpcs[0], nil } +func GetVPCEndpointsByID(e2eCtx *E2EContext, vpcID string) ([]*ec2.VpcEndpoint, error) { + ec2Svc := ec2.New(e2eCtx.AWSSession) + + input := &ec2.DescribeVpcEndpointsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: aws.StringSlice([]string{vpcID}), + }, + }, + } + + res := []*ec2.VpcEndpoint{} + if err := ec2Svc.DescribeVpcEndpointsPages(input, func(dveo *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool { + res = append(res, dveo.VpcEndpoints...) + return true + }); err != nil { + return nil, err + } + + return res, nil +} + func CreateVPC(e2eCtx *E2EContext, vpcName string, cidrBlock string) (*ec2.Vpc, error) { ec2Svc := ec2.New(e2eCtx.AWSSession) diff --git a/test/e2e/suites/unmanaged/unmanaged_functional_test.go b/test/e2e/suites/unmanaged/unmanaged_functional_test.go index fac43c2c56..5237f8d274 100644 --- a/test/e2e/suites/unmanaged/unmanaged_functional_test.go +++ b/test/e2e/suites/unmanaged/unmanaged_functional_test.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/conditions" ) var _ = ginkgo.Context("[unmanaged] [functional]", func() { @@ -1047,7 +1048,7 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { configCluster.ControlPlaneMachineCount = pointer.Int64(1) configCluster.WorkerMachineCount = pointer.Int64(1) configCluster.Flavor = shared.IgnitionFlavor - _, md, _ := createCluster(ctx, configCluster, result) + cluster, md, _ := createCluster(ctx, configCluster, result) workerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), @@ -1062,6 +1063,32 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { }) Expect(len(workerMachines)).To(Equal(1)) Expect(len(controlPlaneMachines)).To(Equal(1)) + + awsCluster, err := GetAWSClusterByName(ctx, namespace.Name, clusterName) + Expect(err).To(BeNil()) + + // Validate that s3 endpoints were created in the vpc. + vpc, err := shared.GetVPCByName(e2eCtx, clusterName+"-vpc") + Expect(err).NotTo(HaveOccurred()) + Expect(vpc).NotTo(BeNil()) + endpoints, err := shared.GetVPCEndpointsByID(e2eCtx, *vpc.VpcId) + Expect(err).NotTo(HaveOccurred()) + Expect(endpoints).NotTo(BeNil()) + Expect(len(endpoints)).To(Equal(1)) + Expect(*endpoints[0].VpcEndpointType).To(Equal("Gateway")) + Expect(*endpoints[0].ServiceName).To(Equal("com.amazonaws." + awsCluster.Spec.Region + ".s3")) + Expect(*endpoints[0].VpcId).To(Equal(*vpc.VpcId)) + + ginkgo.By("Waiting for AWSCluster to show the VPC endpoint as deleted in conditions") + Eventually(func() bool { + awsCluster, err := GetAWSClusterByName(ctx, namespace.Name, clusterName) + Expect(err).To(BeNil()) + return conditions.IsFalse(awsCluster, infrav1.VpcEndpointsReadyCondition) && + conditions.GetReason(awsCluster, infrav1.VpcEndpointsReadyCondition) == clusterv1.DeletedReason + }, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...).Should(BeTrue()) + + ginkgo.By("Deleting the cluster") + deleteCluster(ctx, cluster) }) }) })