From a6a707effbef6f9bb867a1b7cb86dc33d08bbe26 Mon Sep 17 00:00:00 2001 From: epolon Date: Tue, 29 Dec 2020 18:53:46 +0200 Subject: [PATCH 1/5] dont tag ASG security group --- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 07ed5aa60ce32..461d98a3b91fd 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1262,8 +1262,10 @@ export class Cluster extends ClusterBase { autoScalingGroup.role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly')); // EKS Required Tags + // https://docs.aws.amazon.com/eks/latest/userguide/worker.html Tags.of(autoScalingGroup).add(`kubernetes.io/cluster/${this.clusterName}`, 'owned', { applyToLaunchedInstances: true, + excludeResourceTypes: ['AWS::EC2::SecurityGroup'], }); // do not attempt to map the role if `kubectl` is not enabled for this From 6a51ecc61fcb962cf5f4fc46f15ba82166e6ecec Mon Sep 17 00:00:00 2001 From: epolon Date: Tue, 29 Dec 2020 19:28:14 +0200 Subject: [PATCH 2/5] mid work --- packages/@aws-cdk/aws-eks/lib/cluster.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 461d98a3b91fd..9071f1936ea91 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1243,7 +1243,7 @@ export class Cluster extends ClusterBase { autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allIcmp()); // allow traffic to/from managed node groups (eks attaches this security group to the managed nodes) - autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup); + // autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup); const bootstrapEnabled = options.bootstrapEnabled !== undefined ? options.bootstrapEnabled : true; if (options.bootstrapOptions && !bootstrapEnabled) { @@ -1265,6 +1265,8 @@ export class Cluster extends ClusterBase { // https://docs.aws.amazon.com/eks/latest/userguide/worker.html Tags.of(autoScalingGroup).add(`kubernetes.io/cluster/${this.clusterName}`, 'owned', { applyToLaunchedInstances: true, + // exclude security groups to avoid multiple "owned" security groups. + // (the cluster security group already has this tag) excludeResourceTypes: ['AWS::EC2::SecurityGroup'], }); From f684f707bb8e33e90f193205a40a2a9e04517143 Mon Sep 17 00:00:00 2001 From: epolon Date: Tue, 29 Dec 2020 19:51:31 +0200 Subject: [PATCH 3/5] added unit test --- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 +- .../@aws-cdk/aws-eks/test/test.cluster.ts | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 9071f1936ea91..5e7fb32ecacd3 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1243,7 +1243,7 @@ export class Cluster extends ClusterBase { autoScalingGroup.connections.allowToAnyIpv4(ec2.Port.allIcmp()); // allow traffic to/from managed node groups (eks attaches this security group to the managed nodes) - // autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup); + autoScalingGroup.addSecurityGroup(this.clusterSecurityGroup); const bootstrapEnabled = options.bootstrapEnabled !== undefined ? options.bootstrapEnabled : true; if (options.bootstrapOptions && !bootstrapEnabled) { diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index fa63e9edc5112..4306d042acf1b 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -138,6 +138,28 @@ export = { }, + 'security group of self-managed asg is not tagged with owned'(test: Test) { + + // GIVEN + const { stack, vpc } = testFixture(); + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + version: CLUSTER_VERSION, + }); + + // WHEN + cluster.addAutoScalingGroupCapacity('self-managed', { + instanceType: new ec2.InstanceType('t2.medium'), + }); + + // make sure the "kubernetes.io/cluster/: owned" tag isn't here. + test.deepEqual(expect(stack).value.Resources.ClusterselfmanagedInstanceSecurityGroup64468C3A.Properties.Tags, [ + { Key: 'Name', Value: 'Stack/Cluster/self-managed' }, + ]); + test.done(); + + }, + 'cluster security group is attached when connecting self-managed nodes'(test: Test) { // GIVEN From 5733b2005c66f9ad620159d71001190c31536a09 Mon Sep 17 00:00:00 2001 From: epolon Date: Wed, 30 Dec 2020 14:12:06 +0200 Subject: [PATCH 4/5] mid work --- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 +- .../test/integ.eks-cluster.expected.json | 436 +++--------------- .../aws-eks/test/integ.eks-cluster.ts | 78 +++- .../aws-eks/test/pinger/function/index.py | 2 +- 4 files changed, 131 insertions(+), 387 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 5e7fb32ecacd3..305316c781bd6 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1267,7 +1267,7 @@ export class Cluster extends ClusterBase { applyToLaunchedInstances: true, // exclude security groups to avoid multiple "owned" security groups. // (the cluster security group already has this tag) - excludeResourceTypes: ['AWS::EC2::SecurityGroup'], + // excludeResourceTypes: ['AWS::EC2::SecurityGroup'], }); // do not attempt to map the role if `kubectl` is not enabled for this diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index f62d6a1fa0d9a..219a191997155 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -3799,7 +3799,7 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "ClustermanifestsimplewebpodC2D35484": { + "ClustermanifestwebservicemanagednamespaceB715E3B3": { "Type": "Custom::AWSCDK-EKS-KubernetesResource", "Properties": { "ServiceToken": { @@ -3808,7 +3808,7 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, - "Manifest": "[{\"kind\":\"Pod\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webpod\",\"labels\":{\"aws.cdk.eks/prune-c8b6a5b3e6f9f4f1aa9dc400a13c96633da4822b2d\":\"\",\"app\":\"simple-web\"}},\"spec\":{\"containers\":[{\"name\":\"simplewebcontainer\",\"image\":\"nginx\",\"ports\":[{\"containerPort\":80}]}]}}]", + "Manifest": "[{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"webservicemanaged-namespace\",\"labels\":{\"aws.cdk.eks/prune-c8a108df7247e01f65024c9409accb351daf97bcd8\":\"\"}}}]", "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3818,7 +3818,7 @@ "Arn" ] }, - "PruneLabel": "aws.cdk.eks/prune-c8b6a5b3e6f9f4f1aa9dc400a13c96633da4822b2d" + "PruneLabel": "aws.cdk.eks/prune-c8a108df7247e01f65024c9409accb351daf97bcd8" }, "DependsOn": [ "ClusterKubectlReadyBarrier200052AF" @@ -3826,7 +3826,7 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "Clustermanifestsimplewebservice4594DB30": { + "Clustermanifestpodwebservicemanaged3EBF4C00": { "Type": "Custom::AWSCDK-EKS-KubernetesResource", "Properties": { "ServiceToken": { @@ -3835,21 +3835,7 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, - "Manifest": { - "Fn::Join": [ - "", - [ - "[{\"kind\":\"Service\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webservice\",\"annotations\":{\"service.beta.kubernetes.io/aws-load-balancer-internal\":\"true\",\"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups\":\"", - { - "Fn::GetAtt": [ - "WebServiceSecurityGroupA556AEB5", - "GroupId" - ] - }, - "\"},\"labels\":{\"aws.cdk.eks/prune-c84c09bc8d75d4cc4d672e0d3872dcdb35f628dc2c\":\"\"}},\"spec\":{\"type\":\"LoadBalancer\",\"ports\":[{\"port\":9000,\"targetPort\":80}],\"selector\":{\"app\":\"simple-web\"}}}]" - ] - ] - }, + "Manifest": "[{\"kind\":\"Pod\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webpodmanaged\",\"labels\":{\"aws.cdk.eks/prune-c8197bcbbb60c760c3cea54c78bb8397174d8ba879\":\"\",\"app\":\"simple-web\"},\"namespace\":\"webservicemanaged-namespace\"},\"spec\":{\"containers\":[{\"name\":\"simplewebcontainer\",\"image\":\"nginx\",\"ports\":[{\"containerPort\":80}]}],\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"eks.amazonaws.com/nodegroup\",\"operator\":\"Exists\"}]}]}}}}}]", "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3859,16 +3845,17 @@ "Arn" ] }, - "PruneLabel": "aws.cdk.eks/prune-c84c09bc8d75d4cc4d672e0d3872dcdb35f628dc2c" + "PruneLabel": "aws.cdk.eks/prune-c8197bcbbb60c760c3cea54c78bb8397174d8ba879" }, "DependsOn": [ - "ClusterKubectlReadyBarrier200052AF" + "ClusterKubectlReadyBarrier200052AF", + "ClustermanifestwebservicemanagednamespaceB715E3B3" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "ClusterwebserviceLoadBalancerAddress38B566FF": { - "Type": "Custom::AWSCDK-EKS-KubernetesObjectValue", + "Clustermanifestservicewebservicemanaged3DE0B628": { + "Type": "Custom::AWSCDK-EKS-KubernetesResource", "Properties": { "ServiceToken": { "Fn::GetAtt": [ @@ -3876,6 +3863,21 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, + "Manifest": { + "Fn::Join": [ + "", + [ + "[{\"kind\":\"Service\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webservicemanaged\",\"annotations\":{\"service.beta.kubernetes.io/aws-load-balancer-internal\":\"true\",\"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups\":\"", + { + "Fn::GetAtt": [ + "WebServiceSecurityGroupwebservicemanaged2D4FB876", + "GroupId" + ] + }, + "\"},\"namespace\":\"webservicemanaged-namespace\",\"labels\":{\"aws.cdk.eks/prune-c82ddcf1ff1773b9c22d45e1dd4255603d55dfec57\":\"\"}},\"spec\":{\"type\":\"LoadBalancer\",\"ports\":[{\"port\":9000,\"targetPort\":80}],\"selector\":{\"app\":\"simple-web\"}}}]" + ] + ] + }, "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3885,14 +3887,11 @@ "Arn" ] }, - "ObjectType": "service", - "ObjectName": "webservice", - "ObjectNamespace": "default", - "JsonPath": ".status.loadBalancer.ingress[0].hostname", - "TimeoutSeconds": 300 + "PruneLabel": "aws.cdk.eks/prune-c82ddcf1ff1773b9c22d45e1dd4255603d55dfec57" }, "DependsOn": [ - "ClusterKubectlReadyBarrier200052AF" + "ClusterKubectlReadyBarrier200052AF", + "Clustermanifestpodwebservicemanaged3EBF4C00" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" @@ -3977,7 +3976,7 @@ }, "/", { - "Ref": "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853S3Bucket3EB15EF2" + "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806" }, "/", { @@ -3987,7 +3986,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853S3VersionKeyD6A244FC" + "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970" } ] } @@ -4000,7 +3999,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853S3VersionKeyD6A244FC" + "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970" } ] } @@ -4022,11 +4021,11 @@ "Arn" ] }, - "referencetoawscdkeksclustertestAssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3Bucket3AA74A74Ref": { - "Ref": "AssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3Bucket81EA5F11" + "referencetoawscdkeksclustertestAssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3Bucket174F3576Ref": { + "Ref": "AssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3Bucket008DBB35" }, - "referencetoawscdkeksclustertestAssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3VersionKey2EF124C2Ref": { - "Ref": "AssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3VersionKey32DED07C" + "referencetoawscdkeksclustertestAssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3VersionKeyE8595856Ref": { + "Ref": "AssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3VersionKey97C3E1A0" }, "referencetoawscdkeksclustertestVpcPrivateSubnet1Subnet32A4EC2ARef": { "Ref": "VpcPrivateSubnet1Subnet536B997A" @@ -4043,17 +4042,17 @@ "ClusterSecurityGroupId" ] }, - "referencetoawscdkeksclustertestAssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3Bucket69155862Ref": { - "Ref": "AssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3Bucket6DACDE73" + "referencetoawscdkeksclustertestAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketB4E9C142Ref": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" }, - "referencetoawscdkeksclustertestAssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3VersionKey0A6CC98ARef": { - "Ref": "AssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3VersionKey015AEA61" + "referencetoawscdkeksclustertestAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKey1C7C1F5FRef": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" }, - "referencetoawscdkeksclustertestAssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3BucketDD492793Ref": { - "Ref": "AssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3Bucket7EE7EA15" + "referencetoawscdkeksclustertestAssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3Bucket8834EE90Ref": { + "Ref": "AssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3Bucket6ABE1927" }, - "referencetoawscdkeksclustertestAssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3VersionKeyD869415CRef": { - "Ref": "AssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3VersionKey6C948E78" + "referencetoawscdkeksclustertestAssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3VersionKey1CADE360Ref": { + "Ref": "AssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3VersionKeyF55A2EA9" }, "referencetoawscdkeksclustertestVpc9A302ADDRef": { "Ref": "Vpc8378EB38" @@ -4296,10 +4295,10 @@ "AWSCDKCfnUtilsProviderCustomResourceProviderRoleFE0EE867" ] }, - "WebServiceSecurityGroupA556AEB5": { + "WebServiceSecurityGroupwebservicemanaged2D4FB876": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupDescription": "aws-cdk-eks-cluster-test/WebServiceSecurityGroup", + "GroupDescription": "aws-cdk-eks-cluster-test/WebServiceSecurityGroupwebservicemanaged", "SecurityGroupEgress": [ { "CidrIp": "0.0.0.0/0", @@ -4312,7 +4311,7 @@ } } }, - "WebServiceSecurityGroupfromawscdkeksclustertestWebServiceSecurityGroup62BA456890005BF0F34B": { + "WebServiceSecurityGroupwebservicemanagedfromawscdkeksclustertestWebServiceSecurityGroupwebservicemanaged6D1355CF9000DBF17FB5": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "IpProtocol": "tcp", @@ -4320,289 +4319,18 @@ "FromPort": 9000, "GroupId": { "Fn::GetAtt": [ - "WebServiceSecurityGroupA556AEB5", + "WebServiceSecurityGroupwebservicemanaged2D4FB876", "GroupId" ] }, "SourceSecurityGroupId": { "Fn::GetAtt": [ - "WebServiceSecurityGroupA556AEB5", + "WebServiceSecurityGroupwebservicemanaged2D4FB876", "GroupId" ] }, "ToPort": 9000 } - }, - "ServicePingerFunctionServiceRole3120191B": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" - ] - ] - } - ] - } - }, - "ServicePingerFunctionADF51BAF": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43S3BucketB43AFE04" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43S3VersionKeyD4B858BC" - } - ] - } - ] - }, - { - "Fn::Select": [ - 1, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43S3VersionKeyD4B858BC" - } - ] - } - ] - } - ] - ] - } - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "ServicePingerFunctionServiceRole3120191B", - "Arn" - ] - }, - "Runtime": "python3.6", - "Timeout": 600, - "VpcConfig": { - "SecurityGroupIds": [ - { - "Fn::GetAtt": [ - "WebServiceSecurityGroupA556AEB5", - "GroupId" - ] - } - ], - "SubnetIds": [ - { - "Ref": "VpcPrivateSubnet1Subnet536B997A" - }, - { - "Ref": "VpcPrivateSubnet2Subnet3788AAA1" - }, - { - "Ref": "VpcPrivateSubnet3SubnetF258B56E" - } - ] - } - }, - "DependsOn": [ - "ServicePingerFunctionServiceRole3120191B" - ] - }, - "ServicePingerProviderframeworkonEventServiceRole3DB083B7": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - } - ] - } - }, - "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "lambda:InvokeFunction", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "ServicePingerFunctionADF51BAF", - "Arn" - ] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7", - "Roles": [ - { - "Ref": "ServicePingerProviderframeworkonEventServiceRole3DB083B7" - } - ] - } - }, - "ServicePingerProviderframeworkonEventEC59DE20": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" - } - ] - } - ] - }, - { - "Fn::Select": [ - 1, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" - } - ] - } - ] - } - ] - ] - } - }, - "Handler": "framework.onEvent", - "Role": { - "Fn::GetAtt": [ - "ServicePingerProviderframeworkonEventServiceRole3DB083B7", - "Arn" - ] - }, - "Runtime": "nodejs10.x", - "Description": "AWS CDK resource provider framework - onEvent (aws-cdk-eks-cluster-test/ServicePinger/Provider)", - "Environment": { - "Variables": { - "USER_ON_EVENT_FUNCTION_ARN": { - "Fn::GetAtt": [ - "ServicePingerFunctionADF51BAF", - "Arn" - ] - } - } - }, - "Timeout": 900 - }, - "DependsOn": [ - "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7", - "ServicePingerProviderframeworkonEventServiceRole3DB083B7" - ] - }, - "ServicePinger01F6DA06": { - "Type": "AWS::CloudFormation::CustomResource", - "Properties": { - "ServiceToken": { - "Fn::GetAtt": [ - "ServicePingerProviderframeworkonEventEC59DE20", - "Arn" - ] - }, - "Url": { - "Fn::Join": [ - "", - [ - "http://", - { - "Fn::GetAtt": [ - "ClusterwebserviceLoadBalancerAddress38B566FF", - "Value" - ] - }, - ":9000" - ] - ] - } - }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" } }, "Outputs": { @@ -4646,14 +4374,6 @@ ] } }, - "Response": { - "Value": { - "Fn::GetAtt": [ - "ServicePinger01F6DA06", - "Value" - ] - } - }, "ClusterEndpoint": { "Value": { "Fn::GetAtt": [ @@ -4725,41 +4445,41 @@ "Type": "String", "Description": "Artifact hash for asset \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" }, - "AssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3Bucket81EA5F11": { + "AssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3Bucket008DBB35": { "Type": "String", - "Description": "S3 bucket for asset \"d01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34f\"" + "Description": "S3 bucket for asset \"bafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757\"" }, - "AssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fS3VersionKey32DED07C": { + "AssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757S3VersionKey97C3E1A0": { "Type": "String", - "Description": "S3 key for asset version \"d01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34f\"" + "Description": "S3 key for asset version \"bafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757\"" }, - "AssetParametersd01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34fArtifactHashE68669BA": { + "AssetParametersbafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757ArtifactHashF584A7D8": { "Type": "String", - "Description": "Artifact hash for asset \"d01b2d8959358117de0017e6f18135905e5680cfc8a83e406229c02671c2b34f\"" + "Description": "Artifact hash for asset \"bafd50ae9f214e496ff8c72c6425f93dca3ccd590e20963706d5d610d9c75757\"" }, - "AssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3Bucket6DACDE73": { + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { "Type": "String", - "Description": "S3 bucket for asset \"efd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1\"" + "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1S3VersionKey015AEA61": { + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { "Type": "String", - "Description": "S3 key for asset version \"efd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1\"" + "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersefd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1ArtifactHashC9FD06BA": { + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { "Type": "String", - "Description": "Artifact hash for asset \"efd72738f046105c96299fb31b3da40320e71ee9cf74bc37720042898403e2a1\"" + "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3Bucket7EE7EA15": { + "AssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3Bucket6ABE1927": { "Type": "String", - "Description": "S3 bucket for asset \"b61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449f\"" + "Description": "S3 bucket for asset \"844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0\"" }, - "AssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fS3VersionKey6C948E78": { + "AssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0S3VersionKeyF55A2EA9": { "Type": "String", - "Description": "S3 key for asset version \"b61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449f\"" + "Description": "S3 key for asset version \"844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0\"" }, - "AssetParametersb61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449fArtifactHash7E705796": { + "AssetParameters844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0ArtifactHash1D7A2D6E": { "Type": "String", - "Description": "Artifact hash for asset \"b61858bbf1a0be803552e3efa9647befd728156696dff1b413b7b2fd4da1449f\"" + "Description": "Artifact hash for asset \"844c1a4b13479b359ea0e607dccb4a04b73e22cf88cf9b64feed2c5f0de213c0\"" }, "AssetParametersb075459e6bf309093fbd4b9a9e576a5f172b91c14d84eedb0f069566f6abb0deS3Bucket14156880": { "Type": "String", @@ -4785,18 +4505,6 @@ "Type": "String", "Description": "Artifact hash for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\"" }, - "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43S3BucketB43AFE04": { - "Type": "String", - "Description": "S3 bucket for asset \"2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43\"" - }, - "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43S3VersionKeyD4B858BC": { - "Type": "String", - "Description": "S3 key for asset version \"2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43\"" - }, - "AssetParameters2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43ArtifactHashC3527E8B": { - "Type": "String", - "Description": "Artifact hash for asset \"2acc31b34c05692ab3ea9831a27e5f241cffb21857e633d8256b8f0ebf5f3f43\"" - }, "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187": { "Type": "String", "Description": "S3 bucket for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" @@ -4809,17 +4517,17 @@ "Type": "String", "Description": "Artifact hash for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" }, - "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853S3Bucket3EB15EF2": { + "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806": { "Type": "String", - "Description": "S3 bucket for asset \"5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853\"" + "Description": "S3 bucket for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" }, - "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853S3VersionKeyD6A244FC": { + "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970": { "Type": "String", - "Description": "S3 key for asset version \"5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853\"" + "Description": "S3 key for asset version \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" }, - "AssetParameters5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853ArtifactHashD763BE57": { + "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8ArtifactHashAAFBAA4D": { "Type": "String", - "Description": "Artifact hash for asset \"5b4a9f125b1d010c96760d55e0fc56362a73e6ca6da3af20a4d13ea27e369853\"" + "Description": "Artifact hash for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" }, "SsmParameterValueawsserviceeksoptimizedami118amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": { "Type": "AWS::SSM::Parameter::Value", diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts index 12c7334bd96a5..09da4b3d4d23b 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts @@ -8,7 +8,7 @@ import * as kplus from 'cdk8s-plus'; import * as constructs from 'constructs'; import * as eks from '../lib'; import * as hello from './hello-k8s'; -import { Pinger } from './pinger/pinger'; +// import { Pinger } from './pinger/pinger'; import { TestStack } from './util'; @@ -71,7 +71,9 @@ class EksClusterStack extends TestStack { this.assertServiceAccount(); - this.assertServiceLoadBalancerAddress(); + this.assertServiceLoadBalancerAddress('managed'); + + // this.assertServiceLoadBalancerAddress('unmanaged'); new CfnOutput(this, 'ClusterEndpoint', { value: this.cluster.clusterEndpoint }); new CfnOutput(this, 'ClusterArn', { value: this.cluster.clusterArn }); @@ -265,33 +267,64 @@ class EksClusterStack extends TestStack { } - private assertServiceLoadBalancerAddress() { + private assertServiceLoadBalancerAddress(nodes: 'managed' | 'unmanaged') { - const serviceName = 'webservice'; + const serviceName = `webservice${nodes}`; + const namespaceName = `${serviceName}-namespace`; + const podName = `webpod${nodes}`; const labels = { app: 'simple-web' }; const containerPort = 80; const servicePort = 9000; - const pingerSecurityGroup = new ec2.SecurityGroup(this, 'WebServiceSecurityGroup', { + // create a dedicated namespace because the 'default' namespace is managed by fargate in this test + const namespace = this.cluster.addManifest(namespaceName, { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { + name: namespaceName, + }, + }); + + const pingerSecurityGroup = new ec2.SecurityGroup(this, `WebServiceSecurityGroup${serviceName}`, { vpc: this.vpc, }); pingerSecurityGroup.addIngressRule(pingerSecurityGroup, ec2.Port.tcp(servicePort), `allow http ${servicePort} access from myself`); - this.cluster.addManifest('simple-web-pod', { + const pod = this.cluster.addManifest(`pod-${serviceName}`, { kind: 'Pod', apiVersion: 'v1', - metadata: { name: 'webpod', labels: labels }, + metadata: { name: podName, labels: labels, namespace: namespaceName }, spec: { containers: [{ name: 'simplewebcontainer', image: 'nginx', ports: [{ containerPort: containerPort }], }], + // allow targeting self-managed nodes as well since they behave + // slightly different than managed ones with respect to network configuration. + affinity: { + nodeAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: 'eks.amazonaws.com/nodegroup', + operator: nodes === 'managed' ? 'Exists' : 'DoesNotExist', + }, + ], + }, + ], + }, + }, + }, }, }); - this.cluster.addManifest('simple-web-service', { + pod.node.addDependency(namespace); + + const service = this.cluster.addManifest(`service-${serviceName}`, { kind: 'Service', apiVersion: 'v1', metadata: { @@ -301,6 +334,7 @@ class EksClusterStack extends TestStack { 'service.beta.kubernetes.io/aws-load-balancer-internal': 'true', 'service.beta.kubernetes.io/aws-load-balancer-extra-security-groups': pingerSecurityGroup.securityGroupId, }, + namespace: namespaceName, }, spec: { type: 'LoadBalancer', @@ -309,21 +343,23 @@ class EksClusterStack extends TestStack { }, }); - const loadBalancerAddress = this.cluster.getServiceLoadBalancerAddress(serviceName); + service.node.addDependency(pod); - // create a resource that hits the load balancer to make sure - // everything is wired properly. - const pinger = new Pinger(this, 'ServicePinger', { - url: `http://${loadBalancerAddress}:${servicePort}`, - securityGroup: pingerSecurityGroup, - vpc: this.vpc, - }); + // const loadBalancerAddress = this.cluster.getServiceLoadBalancerAddress(serviceName, { namespace: namespaceName }); - // this should display a proper nginx response - // Welcome to nginx!... - new CfnOutput(this, 'Response', { - value: pinger.response, - }); + // // create a resource that hits the load balancer to make sure + // // everything is wired properly. + // const pinger = new Pinger(this, `ServicePinger${serviceName}`, { + // url: `http://${loadBalancerAddress}:${servicePort}`, + // securityGroup: pingerSecurityGroup, + // vpc: this.vpc, + // }); + + // // this should display a proper nginx response + // // Welcome to nginx!... + // new CfnOutput(this, `Response${serviceName}`, { + // value: pinger.response, + // }); } } diff --git a/packages/@aws-cdk/aws-eks/test/pinger/function/index.py b/packages/@aws-cdk/aws-eks/test/pinger/function/index.py index fc8db8fa8ba17..e8c0c218a031f 100644 --- a/packages/@aws-cdk/aws-eks/test/pinger/function/index.py +++ b/packages/@aws-cdk/aws-eks/test/pinger/function/index.py @@ -20,5 +20,5 @@ def handler(event, context): # be functioning response = http.request('GET', url, retries=urllib3.Retry(10, backoff_factor=1)) if response.status != 200: - raise RuntimeError(f'Request failed: {status} ({response.reason})') + raise RuntimeError(f'Request failed: {response.status} ({response.reason})') return {'Data': {'Value': response.data.decode('utf-8')}} \ No newline at end of file From 7ba243cf563f7b22ecc2d43ef6b8e61132cce931 Mon Sep 17 00:00:00 2001 From: epolon Date: Thu, 31 Dec 2020 20:54:07 +0200 Subject: [PATCH 5/5] integ tests --- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 +- .../test/integ.eks-cluster.expected.json | 428 +++++++++++++----- .../aws-eks/test/integ.eks-cluster.ts | 78 +--- 3 files changed, 347 insertions(+), 161 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 305316c781bd6..5e7fb32ecacd3 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1267,7 +1267,7 @@ export class Cluster extends ClusterBase { applyToLaunchedInstances: true, // exclude security groups to avoid multiple "owned" security groups. // (the cluster security group already has this tag) - // excludeResourceTypes: ['AWS::EC2::SecurityGroup'], + excludeResourceTypes: ['AWS::EC2::SecurityGroup'], }); // do not attempt to map the role if `kubectl` is not enabled for this diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 219a191997155..230268fdca0f7 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -1471,20 +1471,6 @@ } ], "Tags": [ - { - "Key": { - "Fn::Join": [ - "", - [ - "kubernetes.io/cluster/", - { - "Ref": "Cluster9EE0221C" - } - ] - ] - }, - "Value": "owned" - }, { "Key": "Name", "Value": "aws-cdk-eks-cluster-test/Cluster/Nodes" @@ -1796,20 +1782,6 @@ } ], "Tags": [ - { - "Key": { - "Fn::Join": [ - "", - [ - "kubernetes.io/cluster/", - { - "Ref": "Cluster9EE0221C" - } - ] - ] - }, - "Value": "owned" - }, { "Key": "Name", "Value": "aws-cdk-eks-cluster-test/Cluster/NodesArm" @@ -2121,20 +2093,6 @@ } ], "Tags": [ - { - "Key": { - "Fn::Join": [ - "", - [ - "kubernetes.io/cluster/", - { - "Ref": "Cluster9EE0221C" - } - ] - ] - }, - "Value": "owned" - }, { "Key": "Name", "Value": "aws-cdk-eks-cluster-test/Cluster/BottlerocketNodes" @@ -2460,20 +2418,6 @@ } ], "Tags": [ - { - "Key": { - "Fn::Join": [ - "", - [ - "kubernetes.io/cluster/", - { - "Ref": "Cluster9EE0221C" - } - ] - ] - }, - "Value": "owned" - }, { "Key": "Name", "Value": "aws-cdk-eks-cluster-test/Cluster/spot" @@ -2818,20 +2762,6 @@ } ], "Tags": [ - { - "Key": { - "Fn::Join": [ - "", - [ - "kubernetes.io/cluster/", - { - "Ref": "Cluster9EE0221C" - } - ] - ] - }, - "Value": "owned" - }, { "Key": "Name", "Value": "aws-cdk-eks-cluster-test/Cluster/InferenceInstances" @@ -3799,7 +3729,7 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "ClustermanifestwebservicemanagednamespaceB715E3B3": { + "ClustermanifestsimplewebpodC2D35484": { "Type": "Custom::AWSCDK-EKS-KubernetesResource", "Properties": { "ServiceToken": { @@ -3808,7 +3738,7 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, - "Manifest": "[{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"name\":\"webservicemanaged-namespace\",\"labels\":{\"aws.cdk.eks/prune-c8a108df7247e01f65024c9409accb351daf97bcd8\":\"\"}}}]", + "Manifest": "[{\"kind\":\"Pod\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webpod\",\"labels\":{\"aws.cdk.eks/prune-c8b6a5b3e6f9f4f1aa9dc400a13c96633da4822b2d\":\"\",\"app\":\"simple-web\"}},\"spec\":{\"containers\":[{\"name\":\"simplewebcontainer\",\"image\":\"nginx\",\"ports\":[{\"containerPort\":80}]}]}}]", "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3818,7 +3748,7 @@ "Arn" ] }, - "PruneLabel": "aws.cdk.eks/prune-c8a108df7247e01f65024c9409accb351daf97bcd8" + "PruneLabel": "aws.cdk.eks/prune-c8b6a5b3e6f9f4f1aa9dc400a13c96633da4822b2d" }, "DependsOn": [ "ClusterKubectlReadyBarrier200052AF" @@ -3826,7 +3756,7 @@ "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "Clustermanifestpodwebservicemanaged3EBF4C00": { + "Clustermanifestsimplewebservice4594DB30": { "Type": "Custom::AWSCDK-EKS-KubernetesResource", "Properties": { "ServiceToken": { @@ -3835,7 +3765,21 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, - "Manifest": "[{\"kind\":\"Pod\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webpodmanaged\",\"labels\":{\"aws.cdk.eks/prune-c8197bcbbb60c760c3cea54c78bb8397174d8ba879\":\"\",\"app\":\"simple-web\"},\"namespace\":\"webservicemanaged-namespace\"},\"spec\":{\"containers\":[{\"name\":\"simplewebcontainer\",\"image\":\"nginx\",\"ports\":[{\"containerPort\":80}]}],\"affinity\":{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"eks.amazonaws.com/nodegroup\",\"operator\":\"Exists\"}]}]}}}}}]", + "Manifest": { + "Fn::Join": [ + "", + [ + "[{\"kind\":\"Service\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webservice\",\"annotations\":{\"service.beta.kubernetes.io/aws-load-balancer-internal\":\"true\",\"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups\":\"", + { + "Fn::GetAtt": [ + "WebServiceSecurityGroupA556AEB5", + "GroupId" + ] + }, + "\"},\"labels\":{\"aws.cdk.eks/prune-c84c09bc8d75d4cc4d672e0d3872dcdb35f628dc2c\":\"\"}},\"spec\":{\"type\":\"LoadBalancer\",\"ports\":[{\"port\":9000,\"targetPort\":80}],\"selector\":{\"app\":\"simple-web\"}}}]" + ] + ] + }, "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3845,17 +3789,16 @@ "Arn" ] }, - "PruneLabel": "aws.cdk.eks/prune-c8197bcbbb60c760c3cea54c78bb8397174d8ba879" + "PruneLabel": "aws.cdk.eks/prune-c84c09bc8d75d4cc4d672e0d3872dcdb35f628dc2c" }, "DependsOn": [ - "ClusterKubectlReadyBarrier200052AF", - "ClustermanifestwebservicemanagednamespaceB715E3B3" + "ClusterKubectlReadyBarrier200052AF" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, - "Clustermanifestservicewebservicemanaged3DE0B628": { - "Type": "Custom::AWSCDK-EKS-KubernetesResource", + "ClusterwebserviceLoadBalancerAddress38B566FF": { + "Type": "Custom::AWSCDK-EKS-KubernetesObjectValue", "Properties": { "ServiceToken": { "Fn::GetAtt": [ @@ -3863,21 +3806,6 @@ "Outputs.awscdkeksclustertestawscdkawseksKubectlProviderframeworkonEventC681B49AArn" ] }, - "Manifest": { - "Fn::Join": [ - "", - [ - "[{\"kind\":\"Service\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"webservicemanaged\",\"annotations\":{\"service.beta.kubernetes.io/aws-load-balancer-internal\":\"true\",\"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups\":\"", - { - "Fn::GetAtt": [ - "WebServiceSecurityGroupwebservicemanaged2D4FB876", - "GroupId" - ] - }, - "\"},\"namespace\":\"webservicemanaged-namespace\",\"labels\":{\"aws.cdk.eks/prune-c82ddcf1ff1773b9c22d45e1dd4255603d55dfec57\":\"\"}},\"spec\":{\"type\":\"LoadBalancer\",\"ports\":[{\"port\":9000,\"targetPort\":80}],\"selector\":{\"app\":\"simple-web\"}}}]" - ] - ] - }, "ClusterName": { "Ref": "Cluster9EE0221C" }, @@ -3887,11 +3815,14 @@ "Arn" ] }, - "PruneLabel": "aws.cdk.eks/prune-c82ddcf1ff1773b9c22d45e1dd4255603d55dfec57" + "ObjectType": "service", + "ObjectName": "webservice", + "ObjectNamespace": "default", + "JsonPath": ".status.loadBalancer.ingress[0].hostname", + "TimeoutSeconds": 300 }, "DependsOn": [ - "ClusterKubectlReadyBarrier200052AF", - "Clustermanifestpodwebservicemanaged3EBF4C00" + "ClusterKubectlReadyBarrier200052AF" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" @@ -4295,10 +4226,10 @@ "AWSCDKCfnUtilsProviderCustomResourceProviderRoleFE0EE867" ] }, - "WebServiceSecurityGroupwebservicemanaged2D4FB876": { + "WebServiceSecurityGroupA556AEB5": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupDescription": "aws-cdk-eks-cluster-test/WebServiceSecurityGroupwebservicemanaged", + "GroupDescription": "aws-cdk-eks-cluster-test/WebServiceSecurityGroup", "SecurityGroupEgress": [ { "CidrIp": "0.0.0.0/0", @@ -4311,7 +4242,7 @@ } } }, - "WebServiceSecurityGroupwebservicemanagedfromawscdkeksclustertestWebServiceSecurityGroupwebservicemanaged6D1355CF9000DBF17FB5": { + "WebServiceSecurityGroupfromawscdkeksclustertestWebServiceSecurityGroup62BA456890005BF0F34B": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "IpProtocol": "tcp", @@ -4319,18 +4250,289 @@ "FromPort": 9000, "GroupId": { "Fn::GetAtt": [ - "WebServiceSecurityGroupwebservicemanaged2D4FB876", + "WebServiceSecurityGroupA556AEB5", "GroupId" ] }, "SourceSecurityGroupId": { "Fn::GetAtt": [ - "WebServiceSecurityGroupwebservicemanaged2D4FB876", + "WebServiceSecurityGroupA556AEB5", "GroupId" ] }, "ToPort": 9000 } + }, + "ServicePingerFunctionServiceRole3120191B": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ] + ] + } + ] + } + }, + "ServicePingerFunctionADF51BAF": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3BucketA6642550" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3VersionKeyFEC50F65" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3VersionKeyFEC50F65" + } + ] + } + ] + } + ] + ] + } + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "ServicePingerFunctionServiceRole3120191B", + "Arn" + ] + }, + "Runtime": "python3.6", + "Timeout": 600, + "VpcConfig": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "WebServiceSecurityGroupA556AEB5", + "GroupId" + ] + } + ], + "SubnetIds": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ] + } + }, + "DependsOn": [ + "ServicePingerFunctionServiceRole3120191B" + ] + }, + "ServicePingerProviderframeworkonEventServiceRole3DB083B7": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ServicePingerFunctionADF51BAF", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7", + "Roles": [ + { + "Ref": "ServicePingerProviderframeworkonEventServiceRole3DB083B7" + } + ] + } + }, + "ServicePingerProviderframeworkonEventEC59DE20": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + ] + } + ] + } + ] + ] + } + }, + "Handler": "framework.onEvent", + "Role": { + "Fn::GetAtt": [ + "ServicePingerProviderframeworkonEventServiceRole3DB083B7", + "Arn" + ] + }, + "Runtime": "nodejs10.x", + "Description": "AWS CDK resource provider framework - onEvent (aws-cdk-eks-cluster-test/ServicePinger/Provider)", + "Environment": { + "Variables": { + "USER_ON_EVENT_FUNCTION_ARN": { + "Fn::GetAtt": [ + "ServicePingerFunctionADF51BAF", + "Arn" + ] + } + } + }, + "Timeout": 900 + }, + "DependsOn": [ + "ServicePingerProviderframeworkonEventServiceRoleDefaultPolicyD142E8F7", + "ServicePingerProviderframeworkonEventServiceRole3DB083B7" + ] + }, + "ServicePinger01F6DA06": { + "Type": "AWS::CloudFormation::CustomResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "ServicePingerProviderframeworkonEventEC59DE20", + "Arn" + ] + }, + "Url": { + "Fn::Join": [ + "", + [ + "http://", + { + "Fn::GetAtt": [ + "ClusterwebserviceLoadBalancerAddress38B566FF", + "Value" + ] + }, + ":9000" + ] + ] + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" } }, "Outputs": { @@ -4374,6 +4576,14 @@ ] } }, + "Response": { + "Value": { + "Fn::GetAtt": [ + "ServicePinger01F6DA06", + "Value" + ] + } + }, "ClusterEndpoint": { "Value": { "Fn::GetAtt": [ @@ -4505,6 +4715,18 @@ "Type": "String", "Description": "Artifact hash for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\"" }, + "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3BucketA6642550": { + "Type": "String", + "Description": "S3 bucket for asset \"5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636d\"" + }, + "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3VersionKeyFEC50F65": { + "Type": "String", + "Description": "S3 key for asset version \"5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636d\"" + }, + "AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dArtifactHashBEC87846": { + "Type": "String", + "Description": "Artifact hash for asset \"5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636d\"" + }, "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187": { "Type": "String", "Description": "S3 bucket for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts index 09da4b3d4d23b..12c7334bd96a5 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts @@ -8,7 +8,7 @@ import * as kplus from 'cdk8s-plus'; import * as constructs from 'constructs'; import * as eks from '../lib'; import * as hello from './hello-k8s'; -// import { Pinger } from './pinger/pinger'; +import { Pinger } from './pinger/pinger'; import { TestStack } from './util'; @@ -71,9 +71,7 @@ class EksClusterStack extends TestStack { this.assertServiceAccount(); - this.assertServiceLoadBalancerAddress('managed'); - - // this.assertServiceLoadBalancerAddress('unmanaged'); + this.assertServiceLoadBalancerAddress(); new CfnOutput(this, 'ClusterEndpoint', { value: this.cluster.clusterEndpoint }); new CfnOutput(this, 'ClusterArn', { value: this.cluster.clusterArn }); @@ -267,64 +265,33 @@ class EksClusterStack extends TestStack { } - private assertServiceLoadBalancerAddress(nodes: 'managed' | 'unmanaged') { + private assertServiceLoadBalancerAddress() { - const serviceName = `webservice${nodes}`; - const namespaceName = `${serviceName}-namespace`; - const podName = `webpod${nodes}`; + const serviceName = 'webservice'; const labels = { app: 'simple-web' }; const containerPort = 80; const servicePort = 9000; - // create a dedicated namespace because the 'default' namespace is managed by fargate in this test - const namespace = this.cluster.addManifest(namespaceName, { - apiVersion: 'v1', - kind: 'Namespace', - metadata: { - name: namespaceName, - }, - }); - - const pingerSecurityGroup = new ec2.SecurityGroup(this, `WebServiceSecurityGroup${serviceName}`, { + const pingerSecurityGroup = new ec2.SecurityGroup(this, 'WebServiceSecurityGroup', { vpc: this.vpc, }); pingerSecurityGroup.addIngressRule(pingerSecurityGroup, ec2.Port.tcp(servicePort), `allow http ${servicePort} access from myself`); - const pod = this.cluster.addManifest(`pod-${serviceName}`, { + this.cluster.addManifest('simple-web-pod', { kind: 'Pod', apiVersion: 'v1', - metadata: { name: podName, labels: labels, namespace: namespaceName }, + metadata: { name: 'webpod', labels: labels }, spec: { containers: [{ name: 'simplewebcontainer', image: 'nginx', ports: [{ containerPort: containerPort }], }], - // allow targeting self-managed nodes as well since they behave - // slightly different than managed ones with respect to network configuration. - affinity: { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { - matchExpressions: [ - { - key: 'eks.amazonaws.com/nodegroup', - operator: nodes === 'managed' ? 'Exists' : 'DoesNotExist', - }, - ], - }, - ], - }, - }, - }, }, }); - pod.node.addDependency(namespace); - - const service = this.cluster.addManifest(`service-${serviceName}`, { + this.cluster.addManifest('simple-web-service', { kind: 'Service', apiVersion: 'v1', metadata: { @@ -334,7 +301,6 @@ class EksClusterStack extends TestStack { 'service.beta.kubernetes.io/aws-load-balancer-internal': 'true', 'service.beta.kubernetes.io/aws-load-balancer-extra-security-groups': pingerSecurityGroup.securityGroupId, }, - namespace: namespaceName, }, spec: { type: 'LoadBalancer', @@ -343,23 +309,21 @@ class EksClusterStack extends TestStack { }, }); - service.node.addDependency(pod); - - // const loadBalancerAddress = this.cluster.getServiceLoadBalancerAddress(serviceName, { namespace: namespaceName }); + const loadBalancerAddress = this.cluster.getServiceLoadBalancerAddress(serviceName); - // // create a resource that hits the load balancer to make sure - // // everything is wired properly. - // const pinger = new Pinger(this, `ServicePinger${serviceName}`, { - // url: `http://${loadBalancerAddress}:${servicePort}`, - // securityGroup: pingerSecurityGroup, - // vpc: this.vpc, - // }); + // create a resource that hits the load balancer to make sure + // everything is wired properly. + const pinger = new Pinger(this, 'ServicePinger', { + url: `http://${loadBalancerAddress}:${servicePort}`, + securityGroup: pingerSecurityGroup, + vpc: this.vpc, + }); - // // this should display a proper nginx response - // // Welcome to nginx!... - // new CfnOutput(this, `Response${serviceName}`, { - // value: pinger.response, - // }); + // this should display a proper nginx response + // Welcome to nginx!... + new CfnOutput(this, 'Response', { + value: pinger.response, + }); } }