Add parameter for skipping default security groups (#1416)

Pulumi EKS currently always creates a cluster security group and node
security group.
- The cluster security group gets assigned to the control plane ENIs in
addition to the security group EKS creates (see [AWS
Docs](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html)).
This security group gets an ingress rule from the node security group.
- The node security group gets assigned to `NodeGroup` and `NodeGroupV2`
components that do not specify a custom security group.

Users that either manage the node security themselves or use the
`ManagedNodeGroup` component (uses the EKS created SG) do not need those
default security groups.

This change adds a flag on the cluster (`skipDefaultSecurityGroups`)
that will skip creating those default security groups. Instead.

This introduces a small breaking change, the `clusterSecurityGroup`,
`nodeSecurityGroup` and `clusterIngressRule` outputs are now optional.
The impact of this should be minimal because users that create custom
node groups usually do not use the security groups of the cluster for
that. If they do, they need to add a null check.

Fixes #747

docs/eks-v3-migration.md

@@ -160,3 +160,9 @@ The `NodeGroup` and `NodeGroupV2` components now accept inputs for the following
 - `nodeAssociatePublicIpAddress`
 
 If you're using Go you'll need to adjust your program to handle those types being inputs.
+
+### Default Security Groups can now be disabled
+If you do not need the default cluster and node security groups you can disable those now
+with the `skipDefaultSecurityGroups` flag. Those security groups will not be created when setting that flag to true.
+
+Because of this change, the `clusterSecurityGroup`, `nodeSecurityGroup` and `clusterIngressRule` properties are optional now. If you're using those outputs you'll need to update your code accordingly.

examples/examples_nodejs_test.go

@@ -1001,3 +1001,24 @@ func TestAccClusterAddons(t *testing.T) {
 
 	programTestWithExtraOptions(t, &test, nil)
 }
+
+func TestAccSkipDefaultSecurityGroups(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	test := getJSBaseOptions(t).
+		With(integration.ProgramTestOptions{
+			Dir: path.Join(getCwd(t), "tests", "skip-default-security-groups"),
+			ExtraRuntimeValidation: func(t *testing.T, info integration.RuntimeValidationStackInfo) {
+				utils.RunEKSSmokeTest(t,
+					info.Deployment.Resources,
+					info.Outputs["kubeconfig"],
+				)
+
+				assert.Nil(t, info.Outputs["clusterSecurityGroup"])
+				assert.Nil(t, info.Outputs["nodeSecurityGroup"])
+			},
+		})
+
+	programTestWithExtraOptions(t, &test, nil)
+}

examples/extra-sg/index.ts

@@ -61,7 +61,7 @@ const nodeIngressRule = new aws.ec2.SecurityGroupRule("nodeIngressRule", {
   fromPort: 0,
   toPort: 65535,
   protocol: "tcp",
-  securityGroupId: cluster.nodeSecurityGroup.id,
+  securityGroupId: cluster.nodeSecurityGroup.apply((sg) => sg!.id),
   sourceSecurityGroupId: customSecurityGroup.id,
 });
 
@@ -108,6 +108,6 @@ const ng = new eks.NodeGroupV2("example-mng", {
   amiId: "ami-066e69f6f03b5383e",
   extraNodeSecurityGroups: [
     customSecurityGroup, // Plain type
-    cluster.nodeSecurityGroup, // Input type
+    cluster.nodeSecurityGroup.apply(sg => sg!), // Input type
   ],
 });

examples/tests/migrate-nodegroups/utils.ts

@@ -16,8 +16,8 @@ export function createNodeGroup(
 ): eks.NodeGroup {
     return new eks.NodeGroup(name, {
         cluster: args.cluster,
-        nodeSecurityGroup: args.cluster.nodeSecurityGroup,
-        clusterIngressRule: args.cluster.eksClusterIngressRule,
+        nodeSecurityGroup: args.cluster.nodeSecurityGroup.apply(v => v!),
+        clusterIngressRule: args.cluster.eksClusterIngressRule.apply(v => v!),
         instanceType: args.instanceType,
         nodeAssociatePublicIpAddress: false,
         desiredCapacity: args.desiredCapacity,

examples/tests/skip-default-security-groups/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: skip-default-security-groups
+description: EKS cluster without default security groups
+runtime: nodejs

examples/tests/skip-default-security-groups/README.md

@@ -0,0 +1,3 @@
+# examples/tests/skip-default-security-groups
+
+Tests that the cluster can be created without default security groups

examples/tests/skip-default-security-groups/iam.ts

@@ -0,0 +1,28 @@
+import * as aws from "@pulumi/aws";
+import * as pulumi from "@pulumi/pulumi";
+
+const managedPolicyArns: string[] = [
+    "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
+    "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+    "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
+    "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
+];
+
+// Creates a role and attches the EKS worker node IAM managed policies
+export function createRole(name: string): aws.iam.Role {
+    const role = new aws.iam.Role(name, {
+        assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
+            Service: "ec2.amazonaws.com",
+        }),
+    });
+
+    let counter = 0;
+    for (const policy of managedPolicyArns) {
+        // Create RolePolicyAttachment without returning it.
+        const rpa = new aws.iam.RolePolicyAttachment(`${name}-policy-${counter++}`,
+            { policyArn: policy, role: role },
+        );
+    }
+
+    return role;
+}

examples/tests/skip-default-security-groups/index.ts

@@ -0,0 +1,41 @@
+import * as awsx from "@pulumi/awsx";
+import * as eks from "@pulumi/eks";
+import * as iam from "./iam";
+
+// IAM roles for the node group
+const role = iam.createRole("managed-ng-os");
+
+// Create a new VPC
+const eksVpc = new awsx.ec2.Vpc("managed-ng-os", {
+  enableDnsHostnames: true,
+  cidrBlock: "10.0.0.0/16",
+});
+
+// Create an EKS cluster without default security groups, those are not needed
+// for managed node groups because they use the cluster security group created
+// by EKS.
+const cluster = new eks.Cluster("managed-ng-os", {
+  skipDefaultSecurityGroups: true,
+  vpcId: eksVpc.vpcId,
+  authenticationMode: eks.AuthenticationMode.API,
+  // Public subnets will be used for load balancers
+  publicSubnetIds: eksVpc.publicSubnetIds,
+  // Private subnets will be used for cluster nodes
+  privateSubnetIds: eksVpc.privateSubnetIds,
+});
+
+// Export the cluster's kubeconfig.
+export const kubeconfig = cluster.kubeconfig;
+
+const managedNodeGroupAL2023 = eks.createManagedNodeGroup("al-2023-mng", {
+  scalingConfig: {
+    minSize: 1,
+    maxSize: 1,
+    desiredSize: 1,
+  },
+  cluster: cluster,
+  nodeRole: role,
+});
+
+export const clusterSecurityGroup = cluster.clusterSecurityGroup;
+export const nodeSecurityGroup = cluster.nodeSecurityGroup;

examples/tests/skip-default-security-groups/package.json

@@ -0,0 +1,13 @@
+{
+    "name": "skip-default-security-groups",
+    "devDependencies": {
+        "@types/node": "latest",
+        "typescript": "^4.0.0"
+    },
+    "dependencies": {
+        "@pulumi/awsx": "^2.0.0",
+        "@pulumi/aws": "^6.50.1",
+        "@pulumi/eks": "latest",
+        "@pulumi/pulumi": "^3.0.0"
+    }
+}

examples/tests/skip-default-security-groups/tsconfig.json

@@ -0,0 +1,24 @@
+{
+    "compilerOptions": {
+        "outDir": "bin",
+        "target": "es6",
+        "lib": [
+            "es6"
+        ],
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "declaration": true,
+        "sourceMap": true,
+        "stripInternal": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true,
+        "strictNullChecks": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

nodejs/eks/cluster.ts

@@ -137,7 +137,7 @@ export interface CoreData {
     vpcId: pulumi.Output<string>;
     subnetIds: pulumi.Output<string[]>;
     endpoint: pulumi.Output<string>;
-    clusterSecurityGroup: aws.ec2.SecurityGroup;
+    clusterSecurityGroup?: aws.ec2.SecurityGroup;
     provider: k8s.Provider;
     instanceRoles: pulumi.Output<aws.iam.Role[]>;
     nodeGroupOptions: ClusterNodeGroupOptions;
@@ -504,10 +504,10 @@ export function createCore(
     }
 
     // Create the EKS cluster security group
-    let eksClusterSecurityGroup: aws.ec2.SecurityGroup;
+    let eksClusterSecurityGroup: aws.ec2.SecurityGroup | undefined;
     if (args.clusterSecurityGroup) {
         eksClusterSecurityGroup = args.clusterSecurityGroup;
-    } else {
+    } else if (!args.skipDefaultSecurityGroups) {
         eksClusterSecurityGroup = new aws.ec2.SecurityGroup(
             `${name}-eksClusterSecurityGroup`,
             {
@@ -581,7 +581,9 @@ export function createCore(
             name: args.name,
             roleArn: eksRole.apply((r) => r.arn),
             vpcConfig: {
-                securityGroupIds: [eksClusterSecurityGroup.id],
+                securityGroupIds: eksClusterSecurityGroup
+                    ? [eksClusterSecurityGroup.id]
+                    : undefined,
                 subnetIds: clusterSubnetIds,
                 endpointPrivateAccess: args.endpointPrivateAccess,
                 endpointPublicAccess: args.endpointPublicAccess,
@@ -1513,6 +1515,12 @@ export interface ClusterOptions {
      */
     skipDefaultNodeGroup?: boolean;
 
+    /**
+     * If this toggle is set to true, the EKS cluster will be created without the default node and cluster security groups.
+     * Defaults to false.
+     */
+    skipDefaultSecurityGroups?: boolean;
+
     /**
      * Whether or not to deploy the Kubernetes dashboard to the cluster. If the dashboard is deployed, it can be
      * accessed as follows:
@@ -1877,7 +1885,7 @@ export class Cluster extends pulumi.ComponentResource {
     /**
      * The security group for the EKS cluster.
      */
-    public readonly clusterSecurityGroup: aws.ec2.SecurityGroup;
+    public readonly clusterSecurityGroup: aws.ec2.SecurityGroup | undefined;
 
     /**
      * The service roles used by the EKS cluster.
@@ -1887,12 +1895,12 @@ export class Cluster extends pulumi.ComponentResource {
     /**
      * The security group for the cluster's nodes.
      */
-    public readonly nodeSecurityGroup: aws.ec2.SecurityGroup;
+    public readonly nodeSecurityGroup: aws.ec2.SecurityGroup | undefined;
 
     /**
      * The ingress rule that gives node group access to cluster API server
      */
-    public readonly eksClusterIngressRule: aws.ec2.SecurityGroupRule;
+    public readonly eksClusterIngressRule: aws.ec2.SecurityGroupRule | undefined;
 
     /**
      * The default Node Group configuration, or undefined if `skipDefaultNodeGroup` was specified.
@@ -1978,10 +1986,10 @@ export interface ClusterResult {
     kubeconfig: pulumi.Output<any>;
     kubeconfigJson: pulumi.Output<string>;
     awsProvider?: pulumi.ProviderResource;
-    clusterSecurityGroup: aws.ec2.SecurityGroup;
+    clusterSecurityGroup?: aws.ec2.SecurityGroup;
     instanceRoles: pulumi.Output<aws.iam.Role[]>;
-    nodeSecurityGroup: aws.ec2.SecurityGroup;
-    eksClusterIngressRule: aws.ec2.SecurityGroupRule;
+    nodeSecurityGroup?: aws.ec2.SecurityGroup;
+    eksClusterIngressRule?: aws.ec2.SecurityGroupRule;
     defaultNodeGroup: NodeGroupV2Data | undefined;
     eksCluster: aws.eks.Cluster;
     core: CoreData;
@@ -2019,25 +2027,36 @@ export function createCluster(
     // Create the core resources required by the cluster.
     const core = createCore(name, args, self, opts?.provider);
 
-    // Create default node group security group and cluster ingress rule.
-    const [nodeSecurityGroup, eksClusterIngressRule] = createNodeGroupSecurityGroup(
-        name,
-        {
-            vpcId: core.vpcId,
-            clusterSecurityGroup: core.clusterSecurityGroup,
-            eksCluster: core.cluster,
-            tags: pulumi.all([args.tags, args.nodeSecurityGroupTags]).apply(
-                ([tags, nodeSecurityGroupTags]) =>
-                    <aws.Tags>{
-                        ...nodeSecurityGroupTags,
-                        ...tags,
-                    },
-            ),
-        },
-        self,
-    );
-    core.nodeGroupOptions.nodeSecurityGroup = nodeSecurityGroup;
-    core.nodeGroupOptions.clusterIngressRule = eksClusterIngressRule;
+    let nodeSecurityGroup: aws.ec2.SecurityGroup | undefined;
+    let eksClusterIngressRule: aws.ec2.SecurityGroupRule | undefined;
+    if (!args.skipDefaultSecurityGroups) {
+        if (!core.clusterSecurityGroup) {
+            throw new pulumi.ResourceError(
+                "clusterSecurityGroup is required when creating the default node group.",
+                self,
+            );
+        }
+
+        // Create default node group security group and cluster ingress rule.
+        [nodeSecurityGroup, eksClusterIngressRule] = createNodeGroupSecurityGroup(
+            name,
+            {
+                vpcId: core.vpcId,
+                clusterSecurityGroupId: core.clusterSecurityGroup.id,
+                eksCluster: core.cluster,
+                tags: pulumi.all([args.tags, args.nodeSecurityGroupTags]).apply(
+                    ([tags, nodeSecurityGroupTags]) =>
+                        <aws.Tags>{
+                            ...nodeSecurityGroupTags,
+                            ...tags,
+                        },
+                ),
+            },
+            self,
+        );
+        core.nodeGroupOptions.nodeSecurityGroup = nodeSecurityGroup;
+        core.nodeGroupOptions.clusterIngressRule = eksClusterIngressRule;
+    }
 
     const skipDefaultNodeGroup = args.skipDefaultNodeGroup || args.fargate;
 
@@ -2082,15 +2101,15 @@ export function createCluster(
  * @internal
  */
 export class ClusterInternal extends pulumi.ComponentResource {
-    public readonly clusterSecurityGroup!: pulumi.Output<aws.ec2.SecurityGroup>;
+    public readonly clusterSecurityGroup!: pulumi.Output<aws.ec2.SecurityGroup | undefined>;
     public readonly core!: pulumi.Output<pulumi.Unwrap<CoreData>>;
     public readonly defaultNodeGroup!: pulumi.Output<pulumi.Unwrap<NodeGroupV2Data> | undefined>;
     public readonly eksCluster!: pulumi.Output<aws.eks.Cluster>;
-    public readonly eksClusterIngressRule!: pulumi.Output<aws.ec2.SecurityGroupRule>;
+    public readonly eksClusterIngressRule!: pulumi.Output<aws.ec2.SecurityGroupRule | undefined>;
     public readonly instanceRoles!: pulumi.Output<aws.iam.Role[]>;
     public readonly kubeconfig!: pulumi.Output<any>;
     public readonly kubeconfigJson!: pulumi.Output<string>;
-    public readonly nodeSecurityGroup!: pulumi.Output<aws.ec2.SecurityGroup>;
+    public readonly nodeSecurityGroup!: pulumi.Output<aws.ec2.SecurityGroup | undefined>;
 
     constructor(name: string, args?: ClusterOptions, opts?: pulumi.ComponentResourceOptions) {
         const type = "eks:index:Cluster";