feat(eks): attach cluster security group to self-managed nodes (#12042)

Attaching the EKS managed cluster security group to self managed nodes to allow free traffic flow between managed and self-managed nodes.

Closes #10884

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: iliapolo

packages/@aws-cdk/aws-eks/README.md

@@ -304,6 +304,19 @@ cluster.addAutoScalingGroupCapacity('frontend-nodes', {
 });
 ```
 
+To connect an already initialized auto-scaling group, use the `cluster.connectAutoScalingGroupCapacity()` method:
+
+```ts
+const asg = new ec2.AutoScalingGroup(...);
+cluster.connectAutoScalingGroupCapacity(asg);
+```
+
+In both cases, the [cluster security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html#cluster-sg) will be autoamtically attached to
+the auto-scaling group, allowing for traffic to flow freely between managed and self-managed nodes.
+
+> **Note:** The default `updateType` for auto-scaling groups does not replace existing nodes. Since security groups are determined at launch time, self-managed nodes that were provisioned with version `1.78.0` or lower, will not be updated.
+> To apply the new configuration on all your self-managed nodes, you'll need to replace the nodes using the `UpdateType.REPLACING_UPDATE` policy for the [`updateType`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-autoscaling.AutoScalingGroup.html#updatetypespan-classapi-icon-api-icon-deprecated-titlethis-api-element-is-deprecated-its-use-is-not-recommended%EF%B8%8Fspan) property.
+
 You can customize the [/etc/eks/boostrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) script, which is responsible
 for bootstrapping the node to the EKS cluster. For example, you can use `kubeletExtraArgs` to add custom node labels or taints.
 

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -67,11 +67,17 @@ export interface ICluster extends IResource, ec2.IConnectable {
   readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    * @attribute
    */
   readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   * @attribute
+   */
+  readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    * @attribute
@@ -671,6 +677,7 @@ abstract class ClusterBase extends Resource implements ICluster {
   public abstract readonly clusterEndpoint: string;
   public abstract readonly clusterCertificateAuthorityData: string;
   public abstract readonly clusterSecurityGroupId: string;
+  public abstract readonly clusterSecurityGroup: ec2.ISecurityGroup;
   public abstract readonly clusterEncryptionConfigKeyArn: string;
   public abstract readonly kubectlRole?: iam.IRole;
   public abstract readonly kubectlEnvironment?: { [key: string]: string };
@@ -802,10 +809,15 @@ export class Cluster extends ClusterBase {
   public readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    */
   public readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   */
+  public readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    */
@@ -1070,16 +1082,16 @@ export class Cluster extends ClusterBase {
     this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId;
     this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn;
 
-    const clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
+    this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
 
     this.connections = new ec2.Connections({
-      securityGroups: [clusterSecurityGroup, securityGroup],
+      securityGroups: [this.clusterSecurityGroup, securityGroup],
       defaultPort: ec2.Port.tcp(443), // Control Plane has an HTTPS API
     });
 
     // we can use the cluster security group since its already attached to the cluster
     // and configured to allow connections from itself.
-    this.kubectlSecurityGroup = clusterSecurityGroup;
+    this.kubectlSecurityGroup = this.clusterSecurityGroup;
 
     // use the cluster creation role to issue kubectl commands against the cluster because when the
     // cluster is first created, that's the only role that has "system:masters" permissions
@@ -1254,6 +1266,9 @@ export class Cluster extends ClusterBase {
     autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allUdp());
     autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allIcmp());
 
+    // allow traffic to/from managed node groups (eks attaches this security group to the managed nodes)
+    autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup);
+
     const bootstrapEnabled = options.bootstrapEnabled !== undefined ? options.bootstrapEnabled : true;
     if (options.bootstrapOptions && !bootstrapEnabled) {
       throw new Error('Cannot specify "bootstrapOptions" if "bootstrapEnabled" is false');
@@ -1693,6 +1708,10 @@ class ImportedCluster extends ClusterBase {
   public readonly kubectlMemory?: Size;
   public readonly prune: boolean;
 
+  // so that `clusterSecurityGroup` on `ICluster` can be configured without optionality, avoiding users from having
+  // to null check on an instance of `Cluster`, which will always have this configured.
+  private readonly _clusterSecurityGroup?: ec2.ISecurityGroup;
+
   constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) {
     super(scope, id);
 
@@ -1713,7 +1732,8 @@ class ImportedCluster extends ClusterBase {
     }
 
     if (props.clusterSecurityGroupId) {
-      this.connections.addSecurityGroup(ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', props.clusterSecurityGroupId));
+      this._clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
+      this.connections.addSecurityGroup(this._clusterSecurityGroup);
     }
   }
 
@@ -1724,6 +1744,13 @@ class ImportedCluster extends ClusterBase {
     return this.props.vpc;
   }
 
+  public get clusterSecurityGroup(): ec2.ISecurityGroup {
+    if (!this._clusterSecurityGroup) {
+      throw new Error('"clusterSecurityGroup" is not defined for this imported cluster');
+    }
+    return this._clusterSecurityGroup;
+  }
+
   public get clusterSecurityGroupId(): string {
     if (!this.props.clusterSecurityGroupId) {
       throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster');

packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts

@@ -111,10 +111,16 @@ export class LegacyCluster extends Resource implements ICluster {
   public readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    */
   public readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   */
+  public readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    */
@@ -218,6 +224,7 @@ export class LegacyCluster extends Resource implements ICluster {
     this.clusterEndpoint = resource.attrEndpoint;
     this.clusterCertificateAuthorityData = resource.attrCertificateAuthorityData;
     this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId;
+    this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
     this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn;
 
     const updateConfigCommandPrefix = `aws eks update-kubeconfig --name ${this.clusterName}`;
@@ -468,6 +475,11 @@ class ImportedCluster extends Resource implements ICluster {
     return this.props.vpc;
   }
 
+  public get clusterSecurityGroup(): ec2.ISecurityGroup {
+    // we are getting rid of this class very soon, no real need to support this here.
+    throw new Error("Unsupported operation. Use 'clusterSecurityGroupId' instead.");
+  }
+
   public get clusterSecurityGroupId(): string {
     if (!this.props.clusterSecurityGroupId) {
       throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster');

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json

@@ -1702,6 +1702,12 @@
               "ClusterNodesInstanceSecurityGroup899246BD",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2021,6 +2027,12 @@
               "ClusterNodesArmInstanceSecurityGroup599F388B",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2340,6 +2352,12 @@
               "ClusterBottlerocketNodesInstanceSecurityGroup3794A94B",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2673,6 +2691,12 @@
               "ClusterspotInstanceSecurityGroup01F7B1CE",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "SpotPrice": "0.1094",
@@ -3025,6 +3049,12 @@
               "ClusterInferenceInstancesInstanceSecurityGroupECB3FC45",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {

packages/@aws-cdk/aws-eks/test/test.cluster.ts

@@ -22,6 +22,88 @@ const CLUSTER_VERSION = eks.KubernetesVersion.V1_18;
 
 export = {
 
+  'throws when accessing cluster security group for imported cluster without cluster security group id'(test: Test) {
+
+    const { stack } = testFixture();
+
+    const cluster = eks.Cluster.fromClusterAttributes(stack, 'Cluster', {
+      clusterName: 'cluster',
+    });
+
+    test.throws(() => cluster.clusterSecurityGroup, /"clusterSecurityGroup" is not defined for this imported cluster/);
+    test.done();
+
+  },
+
+  'can access cluster security group for imported cluster with cluster security group id'(test: Test) {
+
+    const { stack } = testFixture();
+
+    const clusterSgId = 'cluster-sg-id';
+
+    const cluster = eks.Cluster.fromClusterAttributes(stack, 'Cluster', {
+      clusterName: 'cluster',
+      clusterSecurityGroupId: clusterSgId,
+    });
+
+    const clusterSg = cluster.clusterSecurityGroup;
+
+    test.equal(clusterSg.securityGroupId, clusterSgId);
+    test.done();
+  },
+
+  'cluster security group is attached when adding self-managed nodes'(test: Test) {
+
+    // GIVEN
+    const { stack, vpc } = testFixture();
+    const cluster = new eks.Cluster(stack, 'Cluster', {
+      vpc,
+      defaultCapacity: 0,
+      version: CLUSTER_VERSION,
+      prune: false,
+    });
+
+    // WHEN
+    cluster.addAutoScalingGroupCapacity('self-managed', {
+      instanceType: new ec2.InstanceType('t2.medium'),
+    });
+
+    test.deepEqual(expect(stack).value.Resources.ClusterselfmanagedLaunchConfigA5B57EF6.Properties.SecurityGroups, [
+      { 'Fn::GetAtt': ['ClusterselfmanagedInstanceSecurityGroup64468C3A', 'GroupId'] },
+      { 'Fn::GetAtt': ['Cluster9EE0221C', 'ClusterSecurityGroupId'] },
+    ]);
+    test.done();
+
+  },
+
+  'cluster security group is attached when connecting self-managed nodes'(test: Test) {
+
+    // GIVEN
+    const { stack, vpc } = testFixture();
+    const cluster = new eks.Cluster(stack, 'Cluster', {
+      vpc,
+      defaultCapacity: 0,
+      version: CLUSTER_VERSION,
+      prune: false,
+    });
+
+    const selfManaged = new asg.AutoScalingGroup(stack, 'self-managed', {
+      instanceType: new ec2.InstanceType('t2.medium'),
+      vpc: vpc,
+      machineImage: new ec2.AmazonLinuxImage(),
+    });
+
+    // WHEN
+    cluster.connectAutoScalingGroupCapacity(selfManaged, {});
+
+    test.deepEqual(expect(stack).value.Resources.selfmanagedLaunchConfigD41289EB.Properties.SecurityGroups, [
+      { 'Fn::GetAtt': ['selfmanagedInstanceSecurityGroupEA6D80C9', 'GroupId'] },
+      { 'Fn::GetAtt': ['Cluster9EE0221C', 'ClusterSecurityGroupId'] },
+    ]);
+    test.done();
+
+  },
+
   'throws when a non cdk8s chart construct is added as cdk8s chart'(test: Test) {
 
     const { stack } = testFixture();