Skip to content

fix(eks): version update completes prematurely #7526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mergify merged 10 commits into from
Apr 23, 2020
Merged

fix(eks): version update completes prematurely #7526

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
97fe9d1
fix(eks): version update completes prematurely
Apr 22, 2020
258fbe7
update expectation
Apr 23, 2020
1ddfb78
Merge branch 'master' into benisrae/eks-fix-version-update
Apr 23, 2020
aeeb384
custom-resources: formalize the notion of passing arbitrary fields to…
Apr 23, 2020
4f18dc3
use describeUpdate instead of cluster status
Apr 23, 2020
074a7d2
update timeout back to 1m
Apr 23, 2020
6463344
add DescribeUpdate permission
Apr 23, 2020
2de3eff
Merge branch 'master' into benisrae/eks-fix-version-update
Apr 23, 2020
1be46b6
Merge branch 'master' into benisrae/eks-fix-version-update
Apr 23, 2020
d6a0d69
Merge branch 'master' into benisrae/eks-fix-version-update
mergify[bot] Apr 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,36 @@ export class ClusterResourceHandler extends ResourceHandler {
}

await this.eks.updateClusterVersion({ name: this.clusterName, version: newVersion });

// it take a while for the cluster to start the version update, and until
// then the cluster's status is still "ACTIVE", which causes version
// upgrades to complete prematurely. so, we wait here until the cluster
// status changes status from "ACTIVE" and only then yield execution to the
// async waiter. technically the status is expected to be "UPDATING", but it
// is more robust to just make sure it's not "ACTIVE" before we carry on

// wait a total of 5 minutes for this to happen.
let remainingSec = 5 * 60;

while (remainingSec > 0) {
console.log(`waiting for cluster to transition from ACTIVE status (remaining time: ${remainingSec}s)`);
const resp = await this.eks.describeCluster({ name: this.clusterName });
console.log('describeCluster result:', JSON.stringify(resp, undefined, 2));

const status = resp.cluster?.status;
if (status !== 'ACTIVE') {
console.log(`cluster is now in ${status} state`);
break;
}

// wait 2sec before trying again
await sleep(2000);
remainingSec -= 2;
}

if (remainingSec === 0) {
throw new Error('version update failure: cluster did not transition from ACTIVE status after 5 minutes elapsed');
}
}

private async isActive(): Promise<IsCompleteResponse> {
Expand Down Expand Up @@ -227,3 +257,8 @@ function analyzeUpdate(oldProps: Partial<aws.EKS.CreateClusterRequest>, newProps
updateLogging: JSON.stringify(newProps.logging) !== JSON.stringify(oldProps.logging),
};
}

async function sleep(ms: number) {
console.log(`waiting ${ms} milliseconds`);
return new Promise(ok => setTimeout(ok, ms));
}
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export class ClusterResourceProvider extends NestedStack {
description: 'onEvent handler for EKS cluster resource provider',
runtime: HANDLER_RUNTIME,
handler: 'index.onEvent',
timeout: Duration.minutes(1),
timeout: Duration.minutes(15),
});

const isComplete = new lambda.Function(this, 'IsCompleteHandler', {
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-eks/lib/cluster.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1078,6 +1078,6 @@ export enum MachineImageType {

const GPU_INSTANCETYPES = ['p2', 'p3', 'g4'];

export function nodeTypeForInstanceType(instanceType: ec2.InstanceType) {
function nodeTypeForInstanceType(instanceType: ec2.InstanceType) {
return GPU_INSTANCETYPES.includes(instanceType.toString().substring(0, 2)) ? NodeType.GPU : NodeType.STANDARD;
}
84 changes: 42 additions & 42 deletions packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2231,7 +2231,7 @@
},
"/",
{
"Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3Bucket835D19A2"
"Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423"
},
"/",
{
Expand All @@ -2241,7 +2241,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61"
"Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC"
}
]
}
Expand All @@ -2254,7 +2254,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61"
"Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC"
}
]
}
Expand All @@ -2264,17 +2264,17 @@
]
},
"Parameters": {
"referencetoawscdkeksclustertestAssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket57C4C68FRef": {
"Ref": "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket7CCCFC30"
"referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket7AF1B74FRef": {
"Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE"
},
"referencetoawscdkeksclustertestAssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyBB973CE7Ref": {
"Ref": "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyA2A28538"
"referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey2AABFF64Ref": {
"Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156"
},
"referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": {
"Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C"
"referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": {
"Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F"
},
"referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": {
"Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB"
"referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": {
"Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF"
}
}
}
Expand All @@ -2292,7 +2292,7 @@
},
"/",
{
"Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF"
"Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A"
},
"/",
{
Expand All @@ -2302,7 +2302,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4"
"Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E"
}
]
}
Expand All @@ -2315,7 +2315,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4"
"Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E"
}
]
}
Expand All @@ -2331,11 +2331,11 @@
"referencetoawscdkeksclustertestAssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyA18C5C39Ref": {
"Ref": "AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyBED95764"
},
"referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": {
"Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C"
"referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": {
"Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F"
},
"referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": {
"Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB"
"referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": {
"Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF"
}
}
}
Expand Down Expand Up @@ -2413,29 +2413,29 @@
}
},
"Parameters": {
"AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket7CCCFC30": {
"AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE": {
"Type": "String",
"Description": "S3 bucket for asset \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\""
"Description": "S3 bucket for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\""
},
"AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyA2A28538": {
"AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156": {
"Type": "String",
"Description": "S3 key for asset version \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\""
"Description": "S3 key for asset version \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\""
},
"AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8ArtifactHashC0D1FA2A": {
"AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730ArtifactHashE97DF9EF": {
"Type": "String",
"Description": "Artifact hash for asset \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\""
"Description": "Artifact hash for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\""
},
"AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C": {
"AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F": {
"Type": "String",
"Description": "S3 bucket for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\""
"Description": "S3 bucket for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\""
},
"AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB": {
"AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF": {
"Type": "String",
"Description": "S3 key for asset version \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\""
"Description": "S3 key for asset version \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\""
},
"AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1ArtifactHash251241BC": {
"AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984ArtifactHash67844294": {
"Type": "String",
"Description": "Artifact hash for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\""
"Description": "Artifact hash for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\""
},
"AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3Bucket0C3A00C2": {
"Type": "String",
Expand All @@ -2449,29 +2449,29 @@
"Type": "String",
"Description": "Artifact hash for asset \"a6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afb\""
},
"AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3Bucket835D19A2": {
"AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423": {
"Type": "String",
"Description": "S3 bucket for asset \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\""
"Description": "S3 bucket for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\""
},
"AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61": {
"AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC": {
"Type": "String",
"Description": "S3 key for asset version \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\""
"Description": "S3 key for asset version \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\""
},
"AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69ArtifactHash0E7708FD": {
"AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7ArtifactHash7CFF23C4": {
"Type": "String",
"Description": "Artifact hash for asset \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\""
"Description": "Artifact hash for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\""
},
"AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF": {
"AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A": {
"Type": "String",
"Description": "S3 bucket for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\""
"Description": "S3 bucket for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\""
},
"AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4": {
"AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E": {
"Type": "String",
"Description": "S3 key for asset version \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\""
"Description": "S3 key for asset version \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\""
},
"AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcArtifactHash83AE269A": {
"AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bArtifactHashBB584529": {
"Type": "String",
"Description": "Artifact hash for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\""
"Description": "Artifact hash for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\""
},
"SsmParameterValueawsserviceeksoptimizedami114amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
"Type": "AWS::SSM::Parameter::Value<String>",
Expand Down
16 changes: 16 additions & 0 deletions packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,21 @@ export = {
}, {
version: undefined,
}));

let count = 3;
const restore = mocks.client.describeCluster;
mocks.client.describeCluster = async () => ({
cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' },
});
const resp = await handler.onEvent();
mocks.client.describeCluster = restore;
test.equal(resp, undefined);
test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, {
name: 'physical-resource-id',
version: '12.34',
});
test.equal(mocks.actualRequest.createClusterRequest, undefined);
test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING'
test.done();
},

Expand All @@ -377,13 +385,21 @@ export = {
}, {
version: '1.1',
}));

let count = 3;
const restore = mocks.client.describeCluster;
mocks.client.describeCluster = async () => ({
cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' },
});
const resp = await handler.onEvent();
mocks.client.describeCluster = restore;
test.equal(resp, undefined);
test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, {
name: 'physical-resource-id',
version: '2.0',
});
test.equal(mocks.actualRequest.createClusterRequest, undefined);
test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING'
test.done();
},

Expand Down