feat(eks): attach cluster security group to self-managed nodes

packages/@aws-cdk/aws-eks/README.md

@@ -304,6 +304,19 @@ cluster.addAutoScalingGroupCapacity('frontend-nodes', {
 });
 ```
 
+To connect an already initialized auto-scaling group, use the `cluster.connectAutoScalingGroupCapacity` method:
+
+```ts
+const asg = new ec2.AutoScalingGroup(...);
+cluster.addAutoScalingGroupCapacity(asg);
+```
+
+In both cases, the [cluster security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html#cluster-sg) will be autoamtically attached to
+the auto-scaling group, allowing for traffic to flow freely between managed and self-managed nodes.
+
+> **Note:** The default `updateType` for auto-scaling groups does not replace existing nodes. Since security groups are determined at launch time, self-managed nodes that were provisioned with version `1.78.0` or lower, will not be updated.
+> To apply the new configuration on all your self-managed nodes, you'll need to replace the nodes using the `UpdateType.REPLACING_UPDATE` policy for the [`updateType`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-autoscaling.AutoScalingGroup.html#updatetypespan-classapi-icon-api-icon-deprecated-titlethis-api-element-is-deprecated-its-use-is-not-recommended%EF%B8%8Fspan) property.
+
 You can customize the [/etc/eks/boostrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) script, which is responsible
 for bootstrapping the node to the EKS cluster. For example, you can use `kubeletExtraArgs` to add custom node labels or taints.
 

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -67,11 +67,17 @@ export interface ICluster extends IResource, ec2.IConnectable {
   readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    * @attribute
    */
   readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   * @attribute
+   */
+  readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    * @attribute
@@ -671,6 +677,7 @@ abstract class ClusterBase extends Resource implements ICluster {
   public abstract readonly clusterEndpoint: string;
   public abstract readonly clusterCertificateAuthorityData: string;
   public abstract readonly clusterSecurityGroupId: string;
+  public abstract readonly clusterSecurityGroup: ec2.ISecurityGroup;
   public abstract readonly clusterEncryptionConfigKeyArn: string;
   public abstract readonly kubectlRole?: iam.IRole;
   public abstract readonly kubectlEnvironment?: { [key: string]: string };
@@ -802,10 +809,15 @@ export class Cluster extends ClusterBase {
   public readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    */
   public readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   */
+  public readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    */
@@ -1070,16 +1082,16 @@ export class Cluster extends ClusterBase {
     this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId;
     this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn;
 
-    const clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
+    this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
 
     this.connections = new ec2.Connections({
-      securityGroups: [clusterSecurityGroup, securityGroup],
+      securityGroups: [this.clusterSecurityGroup, securityGroup],
       defaultPort: ec2.Port.tcp(443), // Control Plane has an HTTPS API
     });
 
     // we can use the cluster security group since its already attached to the cluster
     // and configured to allow connections from itself.
-    this.kubectlSecurityGroup = clusterSecurityGroup;
+    this.kubectlSecurityGroup = this.clusterSecurityGroup;
 
     // use the cluster creation role to issue kubectl commands against the cluster because when the
     // cluster is first created, that's the only role that has "system:masters" permissions
@@ -1254,6 +1266,9 @@ export class Cluster extends ClusterBase {
     autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allUdp());
     autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allIcmp());
 
+    // allow traffic to/from managed node groups (eks attaches this security group to the managed nodes)
+    autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup);
+
     const bootstrapEnabled = options.bootstrapEnabled !== undefined ? options.bootstrapEnabled : true;
     if (options.bootstrapOptions && !bootstrapEnabled) {
       throw new Error('Cannot specify "bootstrapOptions" if "bootstrapEnabled" is false');
@@ -1693,6 +1708,10 @@ class ImportedCluster extends ClusterBase {
   public readonly kubectlMemory?: Size;
   public readonly prune: boolean;
 
+  // so that `clusterSecurityGroup` on `ICluster` can be configured without optionality, avoiding users from having
+  // to null check on an instance of `Cluster`, which will always have this configured.
+  private readonly _clusterSecurityGroup?: ec2.ISecurityGroup;
+
   constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) {
     super(scope, id);
 
@@ -1713,7 +1732,8 @@ class ImportedCluster extends ClusterBase {
     }
 
     if (props.clusterSecurityGroupId) {
-      this.connections.addSecurityGroup(ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', props.clusterSecurityGroupId));
+      this._clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
+      this.connections.addSecurityGroup(this._clusterSecurityGroup);
     }
   }
 
@@ -1724,6 +1744,13 @@ class ImportedCluster extends ClusterBase {
     return this.props.vpc;
   }
 
+  public get clusterSecurityGroup(): ec2.ISecurityGroup {
+    if (!this._clusterSecurityGroup) {
+      throw new Error('"clusterSecurityGroup" is not defined for this imported cluster');
+    }
+    return this._clusterSecurityGroup;
+  }
+
   public get clusterSecurityGroupId(): string {
     if (!this.props.clusterSecurityGroupId) {
       throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster');

packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts

@@ -111,10 +111,16 @@ export class LegacyCluster extends Resource implements ICluster {
   public readonly clusterCertificateAuthorityData: string;
 
   /**
-   * The cluster security group that was created by Amazon EKS for the cluster.
+   * The id of the cluster security group that was created by Amazon EKS for the cluster.
    */
   public readonly clusterSecurityGroupId: string;
 
+  /**
+   * The cluster security group that was created by Amazon EKS for the cluster.
+   */
+  public readonly clusterSecurityGroup: ec2.ISecurityGroup;
+
+
   /**
    * Amazon Resource Name (ARN) or alias of the customer master key (CMK).
    */
@@ -218,6 +224,7 @@ export class LegacyCluster extends Resource implements ICluster {
     this.clusterEndpoint = resource.attrEndpoint;
     this.clusterCertificateAuthorityData = resource.attrCertificateAuthorityData;
     this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId;
+    this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId);
     this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn;
 
     const updateConfigCommandPrefix = `aws eks update-kubeconfig --name ${this.clusterName}`;
@@ -468,6 +475,11 @@ class ImportedCluster extends Resource implements ICluster {
     return this.props.vpc;
   }
 
+  public get clusterSecurityGroup(): ec2.ISecurityGroup {
+    // we are getting rid of this class very soon, no real need to support this here.
+    throw new Error("Unsupported operation. Use 'clusterSecurityGroupId' instead.");
+  }
+
   public get clusterSecurityGroupId(): string {
     if (!this.props.clusterSecurityGroupId) {
       throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster');

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json

@@ -1701,6 +1701,12 @@
               "ClusterNodesInstanceSecurityGroup899246BD",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2020,6 +2026,12 @@
               "ClusterNodesArmInstanceSecurityGroup599F388B",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2339,6 +2351,12 @@
               "ClusterBottlerocketNodesInstanceSecurityGroup3794A94B",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -2672,6 +2690,12 @@
               "ClusterspotInstanceSecurityGroup01F7B1CE",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "SpotPrice": "0.1094",
@@ -3024,6 +3048,12 @@
               "ClusterInferenceInstancesInstanceSecurityGroupECB3FC45",
               "GroupId"
             ]
+          },
+          {
+            "Fn::GetAtt": [
+              "Cluster9EE0221C",
+              "ClusterSecurityGroupId"
+            ]
           }
         ],
         "UserData": {
@@ -3836,7 +3866,7 @@
               },
               "/",
               {
-                "Ref": "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956S3BucketE0699DCA"
+                "Ref": "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30S3Bucket69D787F5"
               },
               "/",
               {
@@ -3846,7 +3876,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956S3VersionKey09D9D914"
+                        "Ref": "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30S3VersionKey55B0E968"
                       }
                     ]
                   }
@@ -3859,7 +3889,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956S3VersionKey09D9D914"
+                        "Ref": "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30S3VersionKey55B0E968"
                       }
                     ]
                   }
@@ -4601,17 +4631,17 @@
       "Type": "String",
       "Description": "Artifact hash for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\""
     },
-    "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956S3BucketE0699DCA": {
+    "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30S3Bucket69D787F5": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956\""
+      "Description": "S3 bucket for asset \"1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30\""
     },
-    "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956S3VersionKey09D9D914": {
+    "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30S3VersionKey55B0E968": {
       "Type": "String",
-      "Description": "S3 key for asset version \"25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956\""
+      "Description": "S3 key for asset version \"1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30\""
     },
-    "AssetParameters25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956ArtifactHash2F859D25": {
+    "AssetParameters1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30ArtifactHash629F107B": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"25aed688c0803654674984565f9e68dec4fbd4f8427b6e9db1be3c61b4fa2956\""
+      "Description": "Artifact hash for asset \"1ab8e76c32e07d160eead3712369019d8fb041ead8695a111d50469c2fa58c30\""
     },
     "SsmParameterValueawsserviceeksoptimizedami118amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",