diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 479c3f636307a..3e41aa6414d02 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -437,7 +437,7 @@ and might have breaking changes in the future. #### Import existing resources -**Important:** This is a work in progress, only S3 buckets are currently supported +**Important:** This is a work in progress Sometimes, it is beneficial to import (enroll/adopt/...) AWS resources, that were created manually (or by different means), into a CDK stack. Some resources can simply be @@ -452,13 +452,13 @@ To import an existing resource to a CDK stack: - add corresponding constructs for the resources to be added in your stack - for example, for an S3 bucket, add something like `new s3.Bucket(this, 'ImportedS3Bucket', {});` - **no other changes must be done to the stack before the import is completed** -- run `cdk deploy` with `--import-resources` argument to instruct CDK to start the import - operation -- if resource definition contains all information needed for the import, this happens +- run `cdk import` command - if there are multiple stacks in the CDK app, pass a specific + stack name as an argument +- if the resource definition contains all information needed for the import, this happens automatically (e.g. an `s3.Bucket` construct has an explicit `bucketName` set), otherwise, CDK will prompt user to provide neccessary identification information (e.g. the bucket name) -- after cdk deploy reports success, the resource is managed by CDK. Any subsequent +- after cdk import reports success, the resource is managed by CDK. Any subsequent changes in the construct configuration will be reflected on the resource diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 0a2325bb41173..e8d1aec382227 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -124,8 +124,16 @@ async function parseCommandLineArguments() { desc: 'Continuously observe the project files, ' + 'and deploy the given stack(s) automatically when changes are detected. ' + 'Implies --hotswap by default', - }) - .option('import-resources', { type: 'boolean', alias: 'i', desc: 'Import existing resources in the stack' }), + }), + ) + .command('import [STACK]', 'Import existing resource(s) into the given STACK', yargs => yargs + .option('execute', { type: 'boolean', desc: 'Whether to execute ChangeSet (--no-execute will NOT execute the ChangeSet)', default: true }) + .option('change-set-name', { type: 'string', desc: 'Name of the CloudFormation change set to create' }) + .option('rollback', { + type: 'boolean', + desc: "Rollback stack to stable state on failure. Defaults to 'true', iterate more rapidly with --no-rollback or -R. " + + 'Note: do **not** disable this flag for deployments with resource replacements, as that will always fail', + }), ) .command('watch [STACKS..]', "Shortcut for 'deploy --watch'", yargs => yargs // I'm fairly certain none of these options, present for 'deploy', make sense for 'watch': @@ -376,7 +384,17 @@ async function initCommandLine() { rollback: configuration.settings.get(['rollback']), hotswap: args.hotswap, watch: args.watch, - importResources: args['import-resources'], + }); + + case 'import': + return cli.import({ + selector, + toolkitStackName, + roleArn: args.roleArn, + execute: args.execute, + changeSetName: args.changeSetName, + progress: configuration.settings.get(['progress']), + rollback: configuration.settings.get(['rollback']), }); case 'watch': diff --git a/packages/aws-cdk/lib/api/cloudformation-deployments.ts b/packages/aws-cdk/lib/api/cloudformation-deployments.ts index f3d6cbe0f439d..6a20e5c4912c1 100644 --- a/packages/aws-cdk/lib/api/cloudformation-deployments.ts +++ b/packages/aws-cdk/lib/api/cloudformation-deployments.ts @@ -6,7 +6,7 @@ import { publishAssets } from '../util/asset-publishing'; import { Mode, SdkProvider } from './aws-auth'; import { deployStack, DeployStackResult, destroyStack } from './deploy-stack'; import { ToolkitInfo } from './toolkit-info'; -import { CloudFormationStack, Template, ResourcesToImport } from './util/cloudformation'; +import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries } from './util/cloudformation'; import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; /** @@ -198,13 +198,23 @@ export class CloudFormationDeployments { return stack.template(); } + public async getTemplateSummary(stackArtifact: cxapi.CloudFormationStackArtifact): Promise { + debug(`Retrieving template summary for stack ${stackArtifact.displayName}.`); + const { stackSdk } = await this.prepareSdkFor(stackArtifact, undefined, Mode.ForReading); + const cfn = stackSdk.cloudFormation(); + + return CloudFormationStack.templateSummary(cfn, stackArtifact.template); + } + public async deployStack(options: DeployStackOptions): Promise { const { stackSdk, resolvedEnvironment, cloudFormationRoleArn } = await this.prepareSdkFor(options.stack, options.roleArn); const toolkitInfo = await ToolkitInfo.lookup(resolvedEnvironment, stackSdk, options.toolkitStackName); - // Publish any assets before doing the actual deploy - await this.publishStackAssets(options.stack, toolkitInfo); + // Publish any assets before doing the actual deploy (do not publish any assets on import operation) + if (options.resourcesToImport === undefined) { + await this.publishStackAssets(options.stack, toolkitInfo); + } // Do a verification of the bootstrap stack version await this.validateBootstrapStackVersion( diff --git a/packages/aws-cdk/lib/api/util/cloudformation.ts b/packages/aws-cdk/lib/api/util/cloudformation.ts index 846add6c46138..3dfcb1e9f5cae 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation.ts @@ -17,6 +17,7 @@ interface TemplateParameter { } export type ResourcesToImport = CloudFormation.ResourcesToImport; +export type ResourceIdentifierSummaries = CloudFormation.ResourceIdentifierSummaries; /** * Represents an (existing) Stack in CloudFormation @@ -37,6 +38,17 @@ export class CloudFormationStack { } } + /** + * Retrieve the stack template's summary with the information about resource import identifiers + */ + public static async templateSummary(cfn: CloudFormation, template: any): Promise { + const response = await cfn.getTemplateSummary({ TemplateBody: JSON.stringify(template) }).promise(); + if (!response.ResourceIdentifierSummaries) { + debug('GetTemplateSummary API call did not return "ReousrceIdentifierSummaries"'); + } + return response.ResourceIdentifierSummaries ?? []; + } + /** * Return a copy of the given stack that does not exist * diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 9ff0580c40ce9..ef983f8c22880 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -8,13 +8,12 @@ import * as promptly from 'promptly'; import { environmentsFromDescriptors, globEnvironmentsFromStacks, looksLikeGlob } from '../lib/api/cxapp/environments'; import { SdkProvider } from './api/aws-auth'; import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; -import { CloudFormationDeployments } from './api/cloudformation-deployments'; +import { CloudFormationDeployments, DeployStackOptions } from './api/cloudformation-deployments'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; -import { ResourcesToImport } from './api/util/cloudformation'; import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor'; import { printSecurityDiff, printStackDiff, RequireApproval } from './diff'; -import { prepareResourcesToImport } from './import'; +import { ResourceImporter } from './import'; import { data, debug, error, highlight, print, success, warning } from './logging'; import { deserializeStructure } from './serialize'; import { Configuration, PROJECT_CONFIG } from './settings'; @@ -119,11 +118,6 @@ export class CdkToolkit { return this.watch(options); } - // TODO - print more intelligent message - if (options.importResources) { - warning('Import resources flag was set'); - } - const stacks = await this.selectStacksForDeploy(options.selector, options.exclusively, options.cacheCloudAssembly); const requireApproval = options.requireApproval ?? RequireApproval.Broadening; @@ -174,17 +168,6 @@ export class CdkToolkit { continue; } - let resourcesToImport: ResourcesToImport | undefined = undefined; - if (options.importResources) { - const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack); - resourcesToImport = await prepareResourcesToImport(currentTemplate, stack); - - // There's a CloudFormation limitation that on import operation, no other changes are allowed: - // As CDK always changes the CDKMetadata resource with a new value, as a workaround, we override - // the template's metadata with currently deployed version - stack.template.Resources.CDKMetadata = currentTemplate.Resources.CDKMetadata; - } - if (requireApproval !== RequireApproval.Never) { const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack); if (printSecurityDiff(currentTemplate, stack, requireApproval)) { @@ -208,63 +191,25 @@ export class CdkToolkit { tags = tagsForStack(stack); } - try { - const result = await this.props.cloudFormation.deployStack({ - stack, - deployName: stack.stackName, - roleArn: options.roleArn, - toolkitStackName: options.toolkitStackName, - reuseAssets: options.reuseAssets, - notificationArns: options.notificationArns, - tags, - execute: options.execute, - changeSetName: options.changeSetName, - force: options.force, - parameters: Object.assign({}, parameterMap['*'], parameterMap[stack.stackName]), - usePreviousParameters: options.usePreviousParameters, - progress: options.progress, - ci: options.ci, - rollback: options.rollback, - hotswap: options.hotswap, - extraUserAgent: options.extraUserAgent, - resourcesToImport, - }); - - const message = result.noOp - ? ' ✅ %s (no changes)' - : ' ✅ %s'; - - success('\n' + message, stack.displayName); - - if (Object.keys(result.outputs).length > 0) { - print('\nOutputs:'); - - stackOutputs[stack.stackName] = result.outputs; - } - - for (const name of Object.keys(result.outputs).sort()) { - const value = result.outputs[name]; - print('%s.%s = %s', colors.cyan(stack.id), colors.cyan(name), colors.underline(colors.cyan(value))); - } - - print('\nStack ARN:'); - - data(result.stackArn); - } catch (e) { - error('\n ❌ %s failed: %s', colors.bold(stack.displayName), e); - throw e; - } finally { - // If an outputs file has been specified, create the file path and write stack outputs to it once. - // Outputs are written after all stacks have been deployed. If a stack deployment fails, - // all of the outputs from successfully deployed stacks before the failure will still be written. - if (outputsFile) { - fs.ensureFileSync(outputsFile); - await fs.writeJson(outputsFile, stackOutputs, { - spaces: 2, - encoding: 'utf8', - }); - } - } + await this.cfnDeployStack({ + stack, + deployName: stack.stackName, + roleArn: options.roleArn, + toolkitStackName: options.toolkitStackName, + reuseAssets: options.reuseAssets, + notificationArns: options.notificationArns, + tags, + execute: options.execute, + changeSetName: options.changeSetName, + force: options.force, + parameters: Object.assign({}, parameterMap['*'], parameterMap[stack.stackName]), + usePreviousParameters: options.usePreviousParameters, + progress: options.progress, + ci: options.ci, + rollback: options.rollback, + hotswap: options.hotswap, + extraUserAgent: options.extraUserAgent, + }, stackOutputs, outputsFile); } } @@ -347,6 +292,49 @@ export class CdkToolkit { }); } + public async import(options: ImportOptions) { + const stacks = await this.selectStacksForDeploy(options.selector, true, true); + + if (stacks.stackCount > 1) { + throw new Error(`Stack selection is ambiguous, please choose a specific stack for import [${stacks.stackArtifacts.map(x => x.id).join(', ')}]`); + } + + const stack = stacks.stackArtifacts[0]; + + highlight(stack.displayName); + + if (!stack.environment) { + // eslint-disable-next-line max-len + throw new Error(`Stack ${stack.displayName} does not define an environment, and AWS credentials could not be obtained from standard locations or no region was configured.`); + } + + if (Object.keys(stack.template.Resources || {}).length === 0) { // The generated stack has no resources + warning('%s: stack has no resources, skipping import.', colors.bold(stack.displayName)); + return; + } + + const resourceImporter = new ResourceImporter(stack, this.props.cloudFormation); + const resourcesToImport = await resourceImporter.prepareImport(); + + print('%s: importing resources into stack...', colors.bold(stack.displayName)); + + const tags = tagsForStack(stack); + + await this.cfnDeployStack({ + stack, + deployName: stack.stackName, + roleArn: options.roleArn, + toolkitStackName: options.toolkitStackName, + tags, + execute: options.execute, + changeSetName: options.changeSetName, + usePreviousParameters: true, + progress: options.progress, + rollback: options.rollback, + resourcesToImport, + }); + } + public async destroy(options: DestroyOptions) { let stacks = await this.selectStacksForDestroy(options.selector, options.exclusively); @@ -618,6 +606,49 @@ export class CdkToolkit { // just continue - deploy will show the error } } + + private async cfnDeployStack(options: DeployStackOptions, stackOutputs?: { [key: string]: any }, outputsFile?: string) { + try { + const result = await this.props.cloudFormation.deployStack(options); + + const message = result.noOp + ? ' ✅ %s (no changes)' + : ' ✅ %s'; + + success('\n' + message, options.stack.displayName); + + if (Object.keys(result.outputs).length > 0) { + print('\nOutputs:'); + + if (stackOutputs) { + stackOutputs[options.stack.stackName] = result.outputs; + } + } + + for (const name of Object.keys(result.outputs).sort()) { + const value = result.outputs[name]; + print('%s.%s = %s', colors.cyan(options.stack.id), colors.cyan(name), colors.underline(colors.cyan(value))); + } + + print('\nStack ARN:'); + + data(result.stackArn); + } catch (e) { + error('\n ❌ %s failed: %s', colors.bold(options.stack.displayName), e); + throw e; + } finally { + // If an outputs file has been specified, create the file path and write stack outputs to it once. + // Outputs are written after all stacks have been deployed. If a stack deployment fails, + // all of the outputs from successfully deployed stacks before the failure will still be written. + if (stackOutputs && outputsFile) { + fs.ensureFileSync(outputsFile); + await fs.writeJson(outputsFile, stackOutputs, { + spaces: 2, + encoding: 'utf8', + }); + } + } + } } export interface DiffOptions { @@ -676,19 +707,12 @@ export interface DiffOptions { securityOnly?: boolean; } -interface WatchOptions { +interface CfnDeployOptions { /** * Criteria for selecting stacks to deploy */ selector: StackSelector; - /** - * Only select the given stack - * - * @default false - */ - exclusively?: boolean; - /** * Name of the toolkit stack to use/deploy * @@ -701,11 +725,6 @@ interface WatchOptions { */ roleArn?: string; - /** - * Reuse the assets with the given asset IDs - */ - reuseAssets?: string[]; - /** * Optional name to use for the CloudFormation change set. * If not provided, a name will be generated automatically. @@ -713,10 +732,11 @@ interface WatchOptions { changeSetName?: string; /** - * Always deploy, even if templates are identical. - * @default false + * Whether to execute the ChangeSet + * Not providing `execute` parameter will result in execution of ChangeSet + * @default true */ - force?: boolean; + execute?: boolean; /** * Display mode for stack deployment progress. @@ -732,6 +752,26 @@ interface WatchOptions { * @default true */ readonly rollback?: boolean; +} + +interface WatchOptions extends Omit { + /** + * Only select the given stack + * + * @default false + */ + exclusively?: boolean; + + /** + * Reuse the assets with the given asset IDs + */ + reuseAssets?: string[]; + + /** + * Always deploy, even if templates are identical. + * @default false + */ + force?: boolean; /** * Whether to perform a 'hotswap' deployment. @@ -750,7 +790,7 @@ interface WatchOptions { readonly extraUserAgent?: string; } -export interface DeployOptions extends WatchOptions { +export interface DeployOptions extends CfnDeployOptions, WatchOptions { /** * ARNs of SNS topics that CloudFormation will notify with stack related events */ @@ -768,13 +808,6 @@ export interface DeployOptions extends WatchOptions { */ tags?: Tag[]; - /** - * Whether to execute the ChangeSet - * Not providing `execute` parameter will result in execution of ChangeSet - * @default true - */ - execute?: boolean; - /** * Additional parameters for CloudFormation at deploy time * @default {} @@ -818,16 +851,10 @@ export interface DeployOptions extends WatchOptions { * @default true */ readonly cacheCloudAssembly?: boolean; - - /** - * Whether to import matching existing resources for newly defined constructs in the stack, - * rather than creating new ones - * - * @default false - */ - readonly importResources?: boolean; } +export interface ImportOptions extends CfnDeployOptions {} + export interface DestroyOptions { /** * Criteria for selecting stacks to deploy diff --git a/packages/aws-cdk/lib/import.ts b/packages/aws-cdk/lib/import.ts index 83ac32011c99f..aff64fffd0b15 100644 --- a/packages/aws-cdk/lib/import.ts +++ b/packages/aws-cdk/lib/import.ts @@ -1,53 +1,146 @@ import * as cfnDiff from '@aws-cdk/cloudformation-diff'; import * as cxapi from '@aws-cdk/cx-api'; +import { ResourceToImport } from 'aws-sdk/clients/cloudformation'; import * as promptly from 'promptly'; +import { CloudFormationDeployments } from './api/cloudformation-deployments'; import { ResourcesToImport } from './api/util/cloudformation'; +import { debug, print } from './logging'; -// Basic idea: we want to have a structure (ideally auto-generated from CFN definitions) that lists all resource types -// that support importing and for each type, the identification information -// // For each resource that is to be added in the new template: // - look up the identification information for the resource type [if not found, fail "type not supported"] // - look up the physical resource (perhaps using cloud control API?) [if not found, fail "resource to be imported does not exist"] // - assembe and return "resources to import" object to be passed on to changeset creation // // TEST: can we have a CFN changeset that both creates resources and import other resources? -const RESOURCE_IDENTIFIERS: { [key: string]: string[] } = { - 'AWS::S3::Bucket': ['BucketName'], -}; -export async function prepareResourcesToImport(oldTemplate: any, newTemplate: cxapi.CloudFormationStackArtifact): Promise { - const diff = cfnDiff.diffTemplate(oldTemplate, newTemplate.template); +export class ResourceImporter { + private resourceIdentifiers: { [key: string]: string[] } = {}; + private currentTemplate: any; + + constructor(private stack: cxapi.CloudFormationStackArtifact, private cfn: CloudFormationDeployments) {} + + public async prepareImport(): Promise { + this.currentTemplate = await this.cfn.readCurrentTemplate(this.stack); + this.resourceIdentifiers = await this.fetchResourceIdentifiers(); + + // There's a CloudFormation limitation that on import operation, no other changes are allowed: + // Since CDK always changes the CDKMetadata resource with a new value, as a workaround, we override + // the template's metadata with currently deployed version + this.stack.template.Resources.CDKMetadata = this.currentTemplate.Resources.CDKMetadata; - const additions: { [key: string]: cfnDiff.ResourceDifference } = {}; - diff.resources.forEachDifference((id, chg) => { - if (chg.isAddition) { - additions[id] = chg; + const diff = cfnDiff.diffTemplate(this.currentTemplate, this.stack.template); + + // All other update/delete changes will be treated as an error + const updateDeleteChanges = diff.resources.filter(chg => !(chg?.isAddition ?? false)).changes; + if (updateDeleteChanges.length) { + const offendingResources = Object.keys(updateDeleteChanges).map(this.resourceDisplayName); + throw new Error('No resource updates or deletes are allowed on import operation. Make sure to resolve pending changes ' + + `to existing resources, before attempting an import. Updated/deleted resources: ${offendingResources.join(', ')}`); } - }); - const resourcesToImport: ResourcesToImport = []; - for (let [id, chg] of Object.entries(additions)) { - if (chg.newResourceType === undefined || !(chg.newResourceType in RESOURCE_IDENTIFIERS)) { - throw new Error(`Resource ${id} is of type ${chg.newResourceType} that is not supported for import`); + // Resources in the new template, that are not present in the current template, are a potential import candidates + const importCandidates = diff.resources.filter(chg => chg?.isAddition ?? false).changes; + + const resourcesToImport: ResourcesToImport = []; + for (let [logicalId, chg] of Object.entries(importCandidates)) { + debug(`Considering resource ${logicalId} of type ${chg.newResourceType} for import`); + const resourceImportHint = await this.chooseResourceForImport(logicalId, chg); + if (resourceImportHint) { + // resource will be imported + resourcesToImport.push(resourceImportHint); + } else { + // resource wasn't chosen for import - for the import operation to succeed, this resource must be removed from the template + delete this.stack.template.Resources[logicalId]; + } + } + return resourcesToImport; + } + + private async chooseResourceForImport(logicalId: string, chg: cfnDiff.ResourceDifference): Promise { + const resourceName = this.resourceDisplayName(logicalId); + + // Skip resources that do not support importing + if (chg.newResourceType === undefined || !(chg.newResourceType in this.resourceIdentifiers)) { + print(`Skipping import of ${resourceName} - unsupported resource type ${chg.newResourceType}`); + return undefined; } let identifier: { [key: string]: string } = {}; - for (let idpart of RESOURCE_IDENTIFIERS[chg.newResourceType]) { + let autoImport = true; + for (let idpart of this.resourceIdentifiers[chg.newResourceType]) { if (chg.newProperties && (idpart in chg.newProperties)) { identifier[idpart] = chg.newProperties[idpart]; } else { - const displayName : string = newTemplate.template?.Resources?.[id]?.Metadata?.['aws:cdk:path'] ?? id; - identifier[idpart] = await promptly.prompt(`Please enter ${idpart} of ${chg.newResourceType} to import as ${displayName.replace(/\/Resource$/, '')}: `); + autoImport = false; + const response = await promptly.prompt( + `Enter ${idpart} of ${chg.newResourceType} to import as ${resourceName} (leave empty to skip): `, + { default: '' }, + ); + if (!response) { + print(`Skipping import of ${resourceName}`); + return undefined; + } + identifier[idpart] = response; + } + } + + const props = Object.entries(identifier).map(x => `${x[0]}=${x[1]}`).join(', '); + + if (autoImport) { + const importConfirmed = await promptly.confirm( + `Import physical resource ${chg.newResourceType} with [${props}] as ${resourceName} (yes/no) [default: yes]? `, + { default: 'yes' }, + ); + if (!importConfirmed) { + print(`Skipping import of ${resourceName}`); + return undefined; } } - resourcesToImport.push({ - LogicalResourceId: id, + print(`Physical resource of type ${chg.newResourceType} identified by [${props}] will be imported as ${resourceName}`); + + // CloudFormation resource import API requires each resource that is being imported to have an explicit DeletionPolicy set. If the resource + // doesn't have the DeletionPolicy set, inject it with the value of 'Delete' - CloudFormation default. It is only needed to be present during + // the import operation - subsequent deploys drop the option from the template + if (!this.stack.template.Resources[logicalId].DeletionPolicy) { + this.stack.template.Resources[logicalId].DeletionPolicy = 'Delete'; + } + + return { + LogicalResourceId: logicalId, ResourceType: chg.newResourceType, ResourceIdentifier: identifier, - }); + }; } - return resourcesToImport; + + /** + * For the given template, retrieve information for all used resource types: + * - whether the resource type supports import operation + * - what properties uniquely identify the physical resource to be imported + * + * @returns a mapping from a resource type to a list of property names that together identify the resource for import + */ + private async fetchResourceIdentifiers(): Promise<{ [key: string]: string[] }> { + const resourceIdentifierSummaries = await this.cfn.getTemplateSummary(this.stack); + let resourceIdentifiers: { [key: string]: string[] } = {}; + for (let summary of resourceIdentifierSummaries) { + if ('ResourceType' in summary && summary.ResourceType && 'ResourceIdentifiers' in summary && summary.ResourceIdentifiers) { + resourceIdentifiers[summary.ResourceType] = summary.ResourceIdentifiers; + } + } + return resourceIdentifiers; + } + + /** + * CDK generated logical resource IDs are not very user friendly - whenever possible, use the path in the construct tree instead + */ + private resourceDisplayName(logicalId: string): string { + const treePath = this.stack.template?.Resources?.[logicalId]?.Metadata?.['aws:cdk:path']; + if (treePath) { + // remove the L1 construct id for better readability + return treePath.replace(/\/Resource$/, ''); + } + return logicalId; + } } \ No newline at end of file