From 95691240c6601f536cd7b86da452dfe7ab413dca Mon Sep 17 00:00:00 2001 From: Tomas Mazak Date: Tue, 23 Nov 2021 19:28:58 +0100 Subject: [PATCH] feat(cli): Add an option to import existing resources (currently S3 buckets only) This is an initial proposal to support existing resources import into CDK stacks. As a PoC, this PR shows a working solution for S3 buckets. This is achieved by introducing `-i` / `--import-resources` CLI option to the `cdk deploy` command. If specified, the newly added resources will not be created, but attempted to be imported (adopted) instead. If the resource definition contains the full resource identifier, this happens automatically. For resources that can't be identified (e.g. an S3 bucket without an explicit `bucketName`), user will be prompted for the necessary information. --- packages/aws-cdk/README.md | 27 ++++++++++ packages/aws-cdk/bin/cdk.ts | 4 +- .../lib/api/cloudformation-deployments.ts | 8 ++- packages/aws-cdk/lib/api/deploy-stack.ts | 11 +++- .../aws-cdk/lib/api/util/cloudformation.ts | 2 + .../api/util/cloudformation/stack-status.ts | 2 +- packages/aws-cdk/lib/cdk-toolkit.ts | 27 ++++++++++ packages/aws-cdk/lib/import.ts | 53 +++++++++++++++++++ 8 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 packages/aws-cdk/lib/import.ts diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 1bd9e491e34e4..479c3f636307a 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -435,6 +435,33 @@ to turn them off, pass the `--no-hotswap` option when invoking it. **Note**: This command is considered experimental, and might have breaking changes in the future. +#### Import existing resources + +**Important:** This is a work in progress, only S3 buckets are currently supported + +Sometimes, it is beneficial to import (enroll/adopt/...) AWS resources, that were +created manually (or by different means), into a CDK stack. Some resources can simply be +deleted and recreated by CDK, but for others, this is not convenient: Typically stateful +resources like S3 Buckets, DynamoDB tables, etc., cannot be easily deleted without an +impact on the service. + +To import an existing resource to a CDK stack: + +- run a `cdk diff` to ensure there are no pending changes to the CDK stack you want to + import resources into - if there are, apply/discard them first +- add corresponding constructs for the resources to be added in your stack - for example, + for an S3 bucket, add something like `new s3.Bucket(this, 'ImportedS3Bucket', {});` - + **no other changes must be done to the stack before the import is completed** +- run `cdk deploy` with `--import-resources` argument to instruct CDK to start the import + operation +- if resource definition contains all information needed for the import, this happens + automatically (e.g. an `s3.Bucket` construct has an explicit `bucketName` set), + otherwise, CDK will prompt user to provide neccessary identification information (e.g. + the bucket name) +- after cdk deploy reports success, the resource is managed by CDK. Any subsequent + changes in the construct configuration will be reflected on the resource + + ### `cdk destroy` Deletes a stack from it's environment. This will cause the resources in the stack to be destroyed (unless they were diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 3f90dfc8ff3e2..0a2325bb41173 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -124,7 +124,8 @@ async function parseCommandLineArguments() { desc: 'Continuously observe the project files, ' + 'and deploy the given stack(s) automatically when changes are detected. ' + 'Implies --hotswap by default', - }), + }) + .option('import-resources', { type: 'boolean', alias: 'i', desc: 'Import existing resources in the stack' }), ) .command('watch [STACKS..]', "Shortcut for 'deploy --watch'", yargs => yargs // I'm fairly certain none of these options, present for 'deploy', make sense for 'watch': @@ -375,6 +376,7 @@ async function initCommandLine() { rollback: configuration.settings.get(['rollback']), hotswap: args.hotswap, watch: args.watch, + importResources: args['import-resources'], }); case 'watch': diff --git a/packages/aws-cdk/lib/api/cloudformation-deployments.ts b/packages/aws-cdk/lib/api/cloudformation-deployments.ts index fb7c5410faf3d..f3d6cbe0f439d 100644 --- a/packages/aws-cdk/lib/api/cloudformation-deployments.ts +++ b/packages/aws-cdk/lib/api/cloudformation-deployments.ts @@ -6,7 +6,7 @@ import { publishAssets } from '../util/asset-publishing'; import { Mode, SdkProvider } from './aws-auth'; import { deployStack, DeployStackResult, destroyStack } from './deploy-stack'; import { ToolkitInfo } from './toolkit-info'; -import { CloudFormationStack, Template } from './util/cloudformation'; +import { CloudFormationStack, Template, ResourcesToImport } from './util/cloudformation'; import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; /** @@ -152,6 +152,11 @@ export interface DeployStackOptions { * @default - nothing extra is appended to the User-Agent header */ readonly extraUserAgent?: string; + + /** + * List of existing resources to be IMPORTED into the stack, instead of being CREATED + */ + readonly resourcesToImport?: ResourcesToImport; } export interface DestroyStackOptions { @@ -230,6 +235,7 @@ export class CloudFormationDeployments { rollback: options.rollback, hotswap: options.hotswap, extraUserAgent: options.extraUserAgent, + resourcesToImport: options.resourcesToImport, }); } diff --git a/packages/aws-cdk/lib/api/deploy-stack.ts b/packages/aws-cdk/lib/api/deploy-stack.ts index 40719507d6e20..d1c868c457b53 100644 --- a/packages/aws-cdk/lib/api/deploy-stack.ts +++ b/packages/aws-cdk/lib/api/deploy-stack.ts @@ -14,7 +14,7 @@ import { CfnEvaluationException } from './hotswap/evaluate-cloudformation-templa import { ToolkitInfo } from './toolkit-info'; import { changeSetHasNoChanges, CloudFormationStack, TemplateParameters, waitForChangeSet, - waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges, + waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges, ResourcesToImport, } from './util/cloudformation'; import { StackActivityMonitor, StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; @@ -189,6 +189,12 @@ export interface DeployStackOptions { * @default - nothing extra is appended to the User-Agent header */ readonly extraUserAgent?: string; + + /** + * If set, change set of type IMPORT will be created, and resourcesToImport + * passed to it. + */ + readonly resourcesToImport?: ResourcesToImport; } const LARGE_TEMPLATE_SIZE_KB = 50; @@ -294,7 +300,8 @@ async function prepareAndExecuteChangeSet( const changeSet = await cfn.createChangeSet({ StackName: deployName, ChangeSetName: changeSetName, - ChangeSetType: update ? 'UPDATE' : 'CREATE', + ChangeSetType: options.resourcesToImport ? 'IMPORT' : update ? 'UPDATE' : 'CREATE', + ResourcesToImport: options.resourcesToImport, Description: `CDK Changeset for execution ${executionId}`, TemplateBody: bodyParameter.TemplateBody, TemplateURL: bodyParameter.TemplateURL, diff --git a/packages/aws-cdk/lib/api/util/cloudformation.ts b/packages/aws-cdk/lib/api/util/cloudformation.ts index b3b5660d727c4..846add6c46138 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation.ts @@ -16,6 +16,8 @@ interface TemplateParameter { [key: string]: any; } +export type ResourcesToImport = CloudFormation.ResourcesToImport; + /** * Represents an (existing) Stack in CloudFormation * diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts index e26886225eea8..76069868cf99b 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts @@ -34,7 +34,7 @@ export class StackStatus { } get isDeploySuccess(): boolean { - return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE'); + return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE' || this.name === 'IMPORT_COMPLETE'); } public toString(): string { diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index d6e27a8c8e54b..9ff0580c40ce9 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -11,8 +11,10 @@ import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; import { CloudFormationDeployments } from './api/cloudformation-deployments'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; +import { ResourcesToImport } from './api/util/cloudformation'; import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor'; import { printSecurityDiff, printStackDiff, RequireApproval } from './diff'; +import { prepareResourcesToImport } from './import'; import { data, debug, error, highlight, print, success, warning } from './logging'; import { deserializeStructure } from './serialize'; import { Configuration, PROJECT_CONFIG } from './settings'; @@ -117,6 +119,11 @@ export class CdkToolkit { return this.watch(options); } + // TODO - print more intelligent message + if (options.importResources) { + warning('Import resources flag was set'); + } + const stacks = await this.selectStacksForDeploy(options.selector, options.exclusively, options.cacheCloudAssembly); const requireApproval = options.requireApproval ?? RequireApproval.Broadening; @@ -167,6 +174,17 @@ export class CdkToolkit { continue; } + let resourcesToImport: ResourcesToImport | undefined = undefined; + if (options.importResources) { + const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack); + resourcesToImport = await prepareResourcesToImport(currentTemplate, stack); + + // There's a CloudFormation limitation that on import operation, no other changes are allowed: + // As CDK always changes the CDKMetadata resource with a new value, as a workaround, we override + // the template's metadata with currently deployed version + stack.template.Resources.CDKMetadata = currentTemplate.Resources.CDKMetadata; + } + if (requireApproval !== RequireApproval.Never) { const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack); if (printSecurityDiff(currentTemplate, stack, requireApproval)) { @@ -209,6 +227,7 @@ export class CdkToolkit { rollback: options.rollback, hotswap: options.hotswap, extraUserAgent: options.extraUserAgent, + resourcesToImport, }); const message = result.noOp @@ -799,6 +818,14 @@ export interface DeployOptions extends WatchOptions { * @default true */ readonly cacheCloudAssembly?: boolean; + + /** + * Whether to import matching existing resources for newly defined constructs in the stack, + * rather than creating new ones + * + * @default false + */ + readonly importResources?: boolean; } export interface DestroyOptions { diff --git a/packages/aws-cdk/lib/import.ts b/packages/aws-cdk/lib/import.ts new file mode 100644 index 0000000000000..83ac32011c99f --- /dev/null +++ b/packages/aws-cdk/lib/import.ts @@ -0,0 +1,53 @@ +import * as cfnDiff from '@aws-cdk/cloudformation-diff'; +import * as cxapi from '@aws-cdk/cx-api'; +import * as promptly from 'promptly'; +import { ResourcesToImport } from './api/util/cloudformation'; + +// Basic idea: we want to have a structure (ideally auto-generated from CFN definitions) that lists all resource types +// that support importing and for each type, the identification information +// +// For each resource that is to be added in the new template: +// - look up the identification information for the resource type [if not found, fail "type not supported"] +// - look up the physical resource (perhaps using cloud control API?) [if not found, fail "resource to be imported does not exist"] +// - assembe and return "resources to import" object to be passed on to changeset creation +// +// TEST: can we have a CFN changeset that both creates resources and import other resources? +const RESOURCE_IDENTIFIERS: { [key: string]: string[] } = { + 'AWS::S3::Bucket': ['BucketName'], +}; + +export async function prepareResourcesToImport(oldTemplate: any, newTemplate: cxapi.CloudFormationStackArtifact): Promise { + const diff = cfnDiff.diffTemplate(oldTemplate, newTemplate.template); + + const additions: { [key: string]: cfnDiff.ResourceDifference } = {}; + diff.resources.forEachDifference((id, chg) => { + if (chg.isAddition) { + additions[id] = chg; + } + }); + + const resourcesToImport: ResourcesToImport = []; + for (let [id, chg] of Object.entries(additions)) { + if (chg.newResourceType === undefined || !(chg.newResourceType in RESOURCE_IDENTIFIERS)) { + throw new Error(`Resource ${id} is of type ${chg.newResourceType} that is not supported for import`); + } + + let identifier: { [key: string]: string } = {}; + for (let idpart of RESOURCE_IDENTIFIERS[chg.newResourceType]) { + if (chg.newProperties && (idpart in chg.newProperties)) { + identifier[idpart] = chg.newProperties[idpart]; + } else { + const displayName : string = newTemplate.template?.Resources?.[id]?.Metadata?.['aws:cdk:path'] ?? id; + identifier[idpart] = await promptly.prompt(`Please enter ${idpart} of ${chg.newResourceType} to import as ${displayName.replace(/\/Resource$/, '')}: `); + } + } + + resourcesToImport.push({ + LogicalResourceId: id, + ResourceType: chg.newResourceType, + ResourceIdentifier: identifier, + }); + } + + return resourcesToImport; +} \ No newline at end of file