Skip to content

Commit b1383c8

Browse files
fiunchinhofiunchinho
committed
Only manage security groups for ENIs tagged by CAPA
1 parent bf50223 commit b1383c8

File tree

2 files changed

+278
-0
lines changed

2 files changed

+278
-0
lines changed

‎controllers/awsmachine_controller_unit_test.go

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2768,6 +2768,276 @@ func TestAWSMachineReconcilerReconcileDefaultsToLoadBalancerTypeClassic(t *testi
27682768
g.Expect(err).To(BeNil())
27692769
}
27702770

2771+
func TestAWSMachineReconcilerReconcileDoesntAddSecurityGroupsToNonManagedNetworkInterfaces(t *testing.T) {
2772+
g := NewWithT(t)
2773+
2774+
ns := "testns"
2775+
2776+
cp := &kubeadmv1beta1.KubeadmControlPlane{}
2777+
cp.SetName("capi-cp-test-1")
2778+
cp.SetNamespace(ns)
2779+
2780+
ownerCluster := &clusterv1.Cluster{
2781+
ObjectMeta: metav1.ObjectMeta{Name: "capi-test-1", Namespace: ns},
2782+
Spec: clusterv1.ClusterSpec{
2783+
InfrastructureRef: &corev1.ObjectReference{
2784+
Kind: "AWSCluster",
2785+
Name: "capi-test-1", // assuming same name
2786+
Namespace: ns,
2787+
APIVersion: infrav1.GroupVersion.String(),
2788+
},
2789+
ControlPlaneRef: &corev1.ObjectReference{
2790+
Kind: "KubeadmControlPlane",
2791+
Namespace: cp.Namespace,
2792+
Name: cp.Name,
2793+
APIVersion: kubeadmv1beta1.GroupVersion.String(),
2794+
},
2795+
},
2796+
Status: clusterv1.ClusterStatus{
2797+
InfrastructureReady: true,
2798+
},
2799+
}
2800+
2801+
awsCluster := &infrav1.AWSCluster{
2802+
ObjectMeta: metav1.ObjectMeta{
2803+
Name: "capi-test-1",
2804+
Namespace: ns,
2805+
OwnerReferences: []metav1.OwnerReference{
2806+
{
2807+
APIVersion: clusterv1.GroupVersion.String(),
2808+
Kind: "Cluster",
2809+
Name: ownerCluster.Name,
2810+
UID: "1",
2811+
},
2812+
},
2813+
},
2814+
Spec: infrav1.AWSClusterSpec{
2815+
ControlPlaneLoadBalancer: &infrav1.AWSLoadBalancerSpec{
2816+
Scheme: &infrav1.ELBSchemeInternetFacing,
2817+
// `LoadBalancerType` not set (i.e. empty string; must default to attaching instance to classic LB)
2818+
},
2819+
NetworkSpec: infrav1.NetworkSpec{
2820+
Subnets: infrav1.Subnets{
2821+
infrav1.SubnetSpec{
2822+
ID: "subnet-1",
2823+
IsPublic: false,
2824+
},
2825+
infrav1.SubnetSpec{
2826+
IsPublic: false,
2827+
},
2828+
},
2829+
},
2830+
},
2831+
Status: infrav1.AWSClusterStatus{
2832+
Ready: true,
2833+
Network: infrav1.NetworkStatus{
2834+
SecurityGroups: map[infrav1.SecurityGroupRole]infrav1.SecurityGroup{
2835+
infrav1.SecurityGroupControlPlane: {
2836+
ID: "1",
2837+
},
2838+
infrav1.SecurityGroupNode: {
2839+
ID: "2",
2840+
},
2841+
infrav1.SecurityGroupLB: {
2842+
ID: "3",
2843+
},
2844+
},
2845+
},
2846+
},
2847+
}
2848+
2849+
ownerMachine := &clusterv1.Machine{
2850+
ObjectMeta: metav1.ObjectMeta{
2851+
Labels: map[string]string{
2852+
clusterv1.ClusterNameLabel: "capi-test-1",
2853+
clusterv1.MachineControlPlaneLabel: "", // control plane node so that controller tries to register it with LB
2854+
},
2855+
Name: "capi-test-machine",
2856+
Namespace: ns,
2857+
},
2858+
Spec: clusterv1.MachineSpec{
2859+
ClusterName: "capi-test",
2860+
Bootstrap: clusterv1.Bootstrap{
2861+
DataSecretName: aws.String("bootstrap-data"),
2862+
},
2863+
},
2864+
}
2865+
2866+
awsMachine := &infrav1.AWSMachine{
2867+
ObjectMeta: metav1.ObjectMeta{
2868+
Name: "aws-test-7",
2869+
Namespace: ns,
2870+
OwnerReferences: []metav1.OwnerReference{
2871+
{
2872+
APIVersion: clusterv1.GroupVersion.String(),
2873+
Kind: "Machine",
2874+
Name: "capi-test-machine",
2875+
UID: "1",
2876+
},
2877+
},
2878+
},
2879+
Spec: infrav1.AWSMachineSpec{
2880+
InstanceType: "test",
2881+
ProviderID: aws.String("aws://the-zone/two"),
2882+
CloudInit: infrav1.CloudInit{
2883+
SecureSecretsBackend: infrav1.SecretBackendSecretsManager,
2884+
SecretPrefix: "prefix",
2885+
SecretCount: 1000,
2886+
},
2887+
},
2888+
}
2889+
2890+
controllerIdentity := &infrav1.AWSClusterControllerIdentity{
2891+
TypeMeta: metav1.TypeMeta{
2892+
Kind: string(infrav1.ControllerIdentityKind),
2893+
},
2894+
ObjectMeta: metav1.ObjectMeta{
2895+
Name: "default",
2896+
},
2897+
Spec: infrav1.AWSClusterControllerIdentitySpec{
2898+
AWSClusterIdentitySpec: infrav1.AWSClusterIdentitySpec{
2899+
AllowedNamespaces: &infrav1.AllowedNamespaces{},
2900+
},
2901+
},
2902+
}
2903+
2904+
secret := &corev1.Secret{
2905+
ObjectMeta: metav1.ObjectMeta{
2906+
Name: "bootstrap-data",
2907+
Namespace: ns,
2908+
},
2909+
Data: map[string][]byte{
2910+
"value": []byte("shell-script"),
2911+
},
2912+
}
2913+
2914+
fakeClient := fake.NewClientBuilder().WithObjects(ownerCluster, awsCluster, ownerMachine, awsMachine, controllerIdentity, secret, cp).WithStatusSubresource(awsCluster, awsMachine).Build()
2915+
2916+
recorder := record.NewFakeRecorder(10)
2917+
reconciler := &AWSMachineReconciler{
2918+
Client: fakeClient,
2919+
Recorder: recorder,
2920+
}
2921+
2922+
mockCtrl := gomock.NewController(t)
2923+
ec2Mock := mocks.NewMockEC2API(mockCtrl)
2924+
elbMock := mocks.NewMockELBAPI(mockCtrl)
2925+
secretMock := mock_services.NewMockSecretInterface(mockCtrl)
2926+
2927+
cs, err := getClusterScope(*awsCluster)
2928+
g.Expect(err).To(BeNil())
2929+
2930+
ec2Svc := ec2Service.NewService(cs)
2931+
ec2Svc.EC2Client = ec2Mock
2932+
reconciler.ec2ServiceFactory = func(scope scope.EC2Scope) services.EC2Interface {
2933+
return ec2Svc
2934+
}
2935+
2936+
elbSvc := elbService.NewService(cs)
2937+
elbSvc.EC2Client = ec2Mock
2938+
elbSvc.ELBClient = elbMock
2939+
reconciler.elbServiceFactory = func(scope scope.ELBScope) services.ELBInterface {
2940+
return elbSvc
2941+
}
2942+
2943+
reconciler.secretsManagerServiceFactory = func(clusterScope cloud.ClusterScoper) services.SecretInterface {
2944+
return secretMock
2945+
}
2946+
2947+
ec2Mock.EXPECT().DescribeInstancesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeInstancesInput{
2948+
InstanceIds: aws.StringSlice([]string{"two"}),
2949+
})).Return(&ec2.DescribeInstancesOutput{
2950+
Reservations: []*ec2.Reservation{
2951+
{
2952+
Instances: []*ec2.Instance{
2953+
{
2954+
InstanceId: aws.String("two"),
2955+
InstanceType: aws.String("m5.large"),
2956+
SubnetId: aws.String("subnet-1"),
2957+
ImageId: aws.String("ami-1"),
2958+
State: &ec2.InstanceState{
2959+
Name: aws.String(ec2.InstanceStateNameRunning),
2960+
},
2961+
Placement: &ec2.Placement{
2962+
AvailabilityZone: aws.String("thezone"),
2963+
},
2964+
MetadataOptions: &ec2.InstanceMetadataOptionsResponse{
2965+
HttpEndpoint: aws.String(string(infrav1.InstanceMetadataEndpointStateEnabled)),
2966+
HttpPutResponseHopLimit: aws.Int64(1),
2967+
HttpTokens: aws.String(string(infrav1.HTTPTokensStateOptional)),
2968+
InstanceMetadataTags: aws.String(string(infrav1.InstanceMetadataEndpointStateDisabled)),
2969+
},
2970+
},
2971+
},
2972+
},
2973+
},
2974+
}, nil)
2975+
2976+
// Must attach to a classic LB, not another type. Only these mock calls are therefore expected.
2977+
mockedCreateLBCalls(t, elbMock.EXPECT())
2978+
2979+
ec2Mock.EXPECT().DescribeNetworkInterfacesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeNetworkInterfacesInput{Filters: []*ec2.Filter{
2980+
{
2981+
Name: aws.String("attachment.instance-id"),
2982+
Values: aws.StringSlice([]string{"two"}),
2983+
},
2984+
}})).Return(&ec2.DescribeNetworkInterfacesOutput{
2985+
NetworkInterfaces: []*ec2.NetworkInterface{
2986+
{
2987+
NetworkInterfaceId: aws.String("eni-1"),
2988+
Groups: []*ec2.GroupIdentifier{
2989+
{
2990+
GroupId: aws.String("2"),
2991+
},
2992+
{
2993+
GroupId: aws.String("3"),
2994+
},
2995+
{
2996+
GroupId: aws.String("1"),
2997+
},
2998+
},
2999+
TagSet: []*ec2.Tag{
3000+
{
3001+
Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"),
3002+
Value: aws.String("owned"),
3003+
},
3004+
},
3005+
},
3006+
{
3007+
NetworkInterfaceId: aws.String("eni-cilium"),
3008+
Groups: []*ec2.GroupIdentifier{
3009+
{
3010+
GroupId: aws.String("3"),
3011+
},
3012+
},
3013+
TagSet: []*ec2.Tag{
3014+
{
3015+
Key: aws.String("cilium-managed"),
3016+
Value: aws.String("true"),
3017+
},
3018+
},
3019+
},
3020+
}}, nil).MaxTimes(3)
3021+
ec2Mock.EXPECT().DescribeNetworkInterfaceAttributeWithContext(context.TODO(), gomock.Eq(&ec2.DescribeNetworkInterfaceAttributeInput{
3022+
NetworkInterfaceId: aws.String("eni-1"),
3023+
Attribute: aws.String("groupSet"),
3024+
})).Return(&ec2.DescribeNetworkInterfaceAttributeOutput{Groups: []*ec2.GroupIdentifier{{GroupId: aws.String("3")}}}, nil).MaxTimes(1)
3025+
ec2Mock.EXPECT().DescribeNetworkInterfaceAttributeWithContext(context.TODO(), gomock.Eq(&ec2.DescribeNetworkInterfaceAttributeInput{
3026+
NetworkInterfaceId: aws.String("eni-cilium"),
3027+
Attribute: aws.String("groupSet"),
3028+
})).Return(&ec2.DescribeNetworkInterfaceAttributeOutput{Groups: []*ec2.GroupIdentifier{{GroupId: aws.String("3")}}}, nil).MaxTimes(1)
3029+
ec2Mock.EXPECT().ModifyNetworkInterfaceAttributeWithContext(context.TODO(), gomock.Any()).Times(1)
3030+
3031+
_, err = reconciler.Reconcile(ctx, ctrl.Request{
3032+
NamespacedName: client.ObjectKey{
3033+
Namespace: awsMachine.Namespace,
3034+
Name: awsMachine.Name,
3035+
},
3036+
})
3037+
3038+
g.Expect(err).To(BeNil())
3039+
}
3040+
27713041
func createObject(g *WithT, obj client.Object, namespace string) {
27723042
if obj.DeepCopyObject() != nil {
27733043
obj.SetNamespace(namespace)

‎pkg/cloud/services/ec2/instances.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"encoding/base64"
2222
"fmt"
23+
"slices"
2324
"sort"
2425
"strings"
2526

@@ -737,6 +738,13 @@ func (s *Service) UpdateInstanceSecurityGroups(instanceID string, ids []string)
737738
s.scope.Debug("Found ENIs on instance", "number-of-enis", len(enis), "instance-id", instanceID)
738739

739740
for _, eni := range enis {
741+
// Other components like cilium may add ENIs, and we don't want CAPA changing the security groups on those.
742+
if !slices.ContainsFunc(eni.TagSet, func(t *ec2.Tag) bool {
743+
return aws.StringValue(t.Key) == infrav1.ClusterTagKey(s.scope.Name())
744+
}) {
745+
s.scope.Debug("Skipping ENI without cluster tag", "eni-id", *eni.NetworkInterfaceId)
746+
continue
747+
}
740748
if err := s.attachSecurityGroupsToNetworkInterface(ids, aws.StringValue(eni.NetworkInterfaceId)); err != nil {
741749
return errors.Wrapf(err, "failed to modify network interfaces on instance %q", instanceID)
742750
}

0 commit comments

Comments
 (0)