diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index 903636c5d4e89..57b8d64fe1b66 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -304,6 +304,19 @@ cluster.addAutoScalingGroupCapacity('frontend-nodes', { }); ``` +To connect an already initialized auto-scaling group, use the `cluster.connectAutoScalingGroupCapacity()` method: + +```ts +const asg = new ec2.AutoScalingGroup(...); +cluster.connectAutoScalingGroupCapacity(asg); +``` + +In both cases, the [cluster security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html#cluster-sg) will be autoamtically attached to +the auto-scaling group, allowing for traffic to flow freely between managed and self-managed nodes. + +> **Note:** The default `updateType` for auto-scaling groups does not replace existing nodes. Since security groups are determined at launch time, self-managed nodes that were provisioned with version `1.78.0` or lower, will not be updated. +> To apply the new configuration on all your self-managed nodes, you'll need to replace the nodes using the `UpdateType.REPLACING_UPDATE` policy for the [`updateType`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-autoscaling.AutoScalingGroup.html#updatetypespan-classapi-icon-api-icon-deprecated-titlethis-api-element-is-deprecated-its-use-is-not-recommended%EF%B8%8Fspan) property. + You can customize the [/etc/eks/boostrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) script, which is responsible for bootstrapping the node to the EKS cluster. For example, you can use `kubeletExtraArgs` to add custom node labels or taints. diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index fcbc2aaffdf10..d497a854bb8c7 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -67,11 +67,17 @@ export interface ICluster extends IResource, ec2.IConnectable { readonly clusterCertificateAuthorityData: string; /** - * The cluster security group that was created by Amazon EKS for the cluster. + * The id of the cluster security group that was created by Amazon EKS for the cluster. * @attribute */ readonly clusterSecurityGroupId: string; + /** + * The cluster security group that was created by Amazon EKS for the cluster. + * @attribute + */ + readonly clusterSecurityGroup: ec2.ISecurityGroup; + /** * Amazon Resource Name (ARN) or alias of the customer master key (CMK). * @attribute @@ -671,6 +677,7 @@ abstract class ClusterBase extends Resource implements ICluster { public abstract readonly clusterEndpoint: string; public abstract readonly clusterCertificateAuthorityData: string; public abstract readonly clusterSecurityGroupId: string; + public abstract readonly clusterSecurityGroup: ec2.ISecurityGroup; public abstract readonly clusterEncryptionConfigKeyArn: string; public abstract readonly kubectlRole?: iam.IRole; public abstract readonly kubectlEnvironment?: { [key: string]: string }; @@ -802,10 +809,15 @@ export class Cluster extends ClusterBase { public readonly clusterCertificateAuthorityData: string; /** - * The cluster security group that was created by Amazon EKS for the cluster. + * The id of the cluster security group that was created by Amazon EKS for the cluster. */ public readonly clusterSecurityGroupId: string; + /** + * The cluster security group that was created by Amazon EKS for the cluster. + */ + public readonly clusterSecurityGroup: ec2.ISecurityGroup; + /** * Amazon Resource Name (ARN) or alias of the customer master key (CMK). */ @@ -1070,16 +1082,16 @@ export class Cluster extends ClusterBase { this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId; this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn; - const clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId); + this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId); this.connections = new ec2.Connections({ - securityGroups: [clusterSecurityGroup, securityGroup], + securityGroups: [this.clusterSecurityGroup, securityGroup], defaultPort: ec2.Port.tcp(443), // Control Plane has an HTTPS API }); // we can use the cluster security group since its already attached to the cluster // and configured to allow connections from itself. - this.kubectlSecurityGroup = clusterSecurityGroup; + this.kubectlSecurityGroup = this.clusterSecurityGroup; // use the cluster creation role to issue kubectl commands against the cluster because when the // cluster is first created, that's the only role that has "system:masters" permissions @@ -1254,6 +1266,9 @@ export class Cluster extends ClusterBase { autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allUdp()); autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allIcmp()); + // allow traffic to/from managed node groups (eks attaches this security group to the managed nodes) + autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup); + const bootstrapEnabled = options.bootstrapEnabled !== undefined ? options.bootstrapEnabled : true; if (options.bootstrapOptions && !bootstrapEnabled) { throw new Error('Cannot specify "bootstrapOptions" if "bootstrapEnabled" is false'); @@ -1693,6 +1708,10 @@ class ImportedCluster extends ClusterBase { public readonly kubectlMemory?: Size; public readonly prune: boolean; + // so that `clusterSecurityGroup` on `ICluster` can be configured without optionality, avoiding users from having + // to null check on an instance of `Cluster`, which will always have this configured. + private readonly _clusterSecurityGroup?: ec2.ISecurityGroup; + constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) { super(scope, id); @@ -1713,7 +1732,8 @@ class ImportedCluster extends ClusterBase { } if (props.clusterSecurityGroupId) { - this.connections.addSecurityGroup(ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', props.clusterSecurityGroupId)); + this._clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId); + this.connections.addSecurityGroup(this._clusterSecurityGroup); } } @@ -1724,6 +1744,13 @@ class ImportedCluster extends ClusterBase { return this.props.vpc; } + public get clusterSecurityGroup(): ec2.ISecurityGroup { + if (!this._clusterSecurityGroup) { + throw new Error('"clusterSecurityGroup" is not defined for this imported cluster'); + } + return this._clusterSecurityGroup; + } + public get clusterSecurityGroupId(): string { if (!this.props.clusterSecurityGroupId) { throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster'); diff --git a/packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts b/packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts index 2193ec301de3c..bc83da531c638 100644 --- a/packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts @@ -111,10 +111,16 @@ export class LegacyCluster extends Resource implements ICluster { public readonly clusterCertificateAuthorityData: string; /** - * The cluster security group that was created by Amazon EKS for the cluster. + * The id of the cluster security group that was created by Amazon EKS for the cluster. */ public readonly clusterSecurityGroupId: string; + /** + * The cluster security group that was created by Amazon EKS for the cluster. + */ + public readonly clusterSecurityGroup: ec2.ISecurityGroup; + + /** * Amazon Resource Name (ARN) or alias of the customer master key (CMK). */ @@ -218,6 +224,7 @@ export class LegacyCluster extends Resource implements ICluster { this.clusterEndpoint = resource.attrEndpoint; this.clusterCertificateAuthorityData = resource.attrCertificateAuthorityData; this.clusterSecurityGroupId = resource.attrClusterSecurityGroupId; + this.clusterSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId(this, 'ClusterSecurityGroup', this.clusterSecurityGroupId); this.clusterEncryptionConfigKeyArn = resource.attrEncryptionConfigKeyArn; const updateConfigCommandPrefix = `aws eks update-kubeconfig --name ${this.clusterName}`; @@ -468,6 +475,11 @@ class ImportedCluster extends Resource implements ICluster { return this.props.vpc; } + public get clusterSecurityGroup(): ec2.ISecurityGroup { + // we are getting rid of this class very soon, no real need to support this here. + throw new Error("Unsupported operation. Use 'clusterSecurityGroupId' instead."); + } + public get clusterSecurityGroupId(): string { if (!this.props.clusterSecurityGroupId) { throw new Error('"clusterSecurityGroupId" is not defined for this imported cluster'); diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index aa3993f08d7f4..c6ffd1c00444a 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -1702,6 +1702,12 @@ "ClusterNodesInstanceSecurityGroup899246BD", "GroupId" ] + }, + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "ClusterSecurityGroupId" + ] } ], "UserData": { @@ -2021,6 +2027,12 @@ "ClusterNodesArmInstanceSecurityGroup599F388B", "GroupId" ] + }, + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "ClusterSecurityGroupId" + ] } ], "UserData": { @@ -2340,6 +2352,12 @@ "ClusterBottlerocketNodesInstanceSecurityGroup3794A94B", "GroupId" ] + }, + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "ClusterSecurityGroupId" + ] } ], "UserData": { @@ -2673,6 +2691,12 @@ "ClusterspotInstanceSecurityGroup01F7B1CE", "GroupId" ] + }, + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "ClusterSecurityGroupId" + ] } ], "SpotPrice": "0.1094", @@ -3025,6 +3049,12 @@ "ClusterInferenceInstancesInstanceSecurityGroupECB3FC45", "GroupId" ] + }, + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "ClusterSecurityGroupId" + ] } ], "UserData": { diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index d57c18561c2b4..7c23d49a0b2f3 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -22,6 +22,88 @@ const CLUSTER_VERSION = eks.KubernetesVersion.V1_18; export = { + 'throws when accessing cluster security group for imported cluster without cluster security group id'(test: Test) { + + const { stack } = testFixture(); + + const cluster = eks.Cluster.fromClusterAttributes(stack, 'Cluster', { + clusterName: 'cluster', + }); + + test.throws(() => cluster.clusterSecurityGroup, /"clusterSecurityGroup" is not defined for this imported cluster/); + test.done(); + + }, + + 'can access cluster security group for imported cluster with cluster security group id'(test: Test) { + + const { stack } = testFixture(); + + const clusterSgId = 'cluster-sg-id'; + + const cluster = eks.Cluster.fromClusterAttributes(stack, 'Cluster', { + clusterName: 'cluster', + clusterSecurityGroupId: clusterSgId, + }); + + const clusterSg = cluster.clusterSecurityGroup; + + test.equal(clusterSg.securityGroupId, clusterSgId); + test.done(); + }, + + 'cluster security group is attached when adding self-managed nodes'(test: Test) { + + // GIVEN + const { stack, vpc } = testFixture(); + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + prune: false, + }); + + // WHEN + cluster.addAutoScalingGroupCapacity('self-managed', { + instanceType: new ec2.InstanceType('t2.medium'), + }); + + test.deepEqual(expect(stack).value.Resources.ClusterselfmanagedLaunchConfigA5B57EF6.Properties.SecurityGroups, [ + { 'Fn::GetAtt': ['ClusterselfmanagedInstanceSecurityGroup64468C3A', 'GroupId'] }, + { 'Fn::GetAtt': ['Cluster9EE0221C', 'ClusterSecurityGroupId'] }, + ]); + test.done(); + + }, + + 'cluster security group is attached when connecting self-managed nodes'(test: Test) { + + // GIVEN + const { stack, vpc } = testFixture(); + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + prune: false, + }); + + const selfManaged = new asg.AutoScalingGroup(stack, 'self-managed', { + instanceType: new ec2.InstanceType('t2.medium'), + vpc: vpc, + machineImage: new ec2.AmazonLinuxImage(), + }); + + // WHEN + cluster.connectAutoScalingGroupCapacity(selfManaged, {}); + + test.deepEqual(expect(stack).value.Resources.selfmanagedLaunchConfigD41289EB.Properties.SecurityGroups, [ + { 'Fn::GetAtt': ['selfmanagedInstanceSecurityGroupEA6D80C9', 'GroupId'] }, + { 'Fn::GetAtt': ['Cluster9EE0221C', 'ClusterSecurityGroupId'] }, + ]); + test.done(); + + }, + 'throws when a non cdk8s chart construct is added as cdk8s chart'(test: Test) { const { stack } = testFixture();