feat(cli): Add an option to import existing resources (currently S3 buckets only)

This is an initial proposal to support existing resources import into
CDK stacks. As a PoC, this PR shows a working solution for S3 buckets.
This is achieved by introducing `-i` / `--import-resources` CLI option
to the `cdk deploy` command. If specified, the newly added resources
will not be created, but attempted to be imported (adopted) instead. If
the resource definition contains the full resource identifier, this
happens automatically. For resources that can't be identified (e.g. an
S3 bucket without an explicit `bucketName`), user will be prompted for
the necessary information.

Author: tomas-mazak

Diff for: packages/aws-cdk/README.md

@@ -435,6 +435,33 @@ to turn them off, pass the `--no-hotswap` option when invoking it.
 **Note**: This command is considered experimental,
 and might have breaking changes in the future.
 
+#### Import existing resources
+
+**Important:** This is a work in progress, only S3 buckets are currently supported
+
+Sometimes, it is beneficial to import (enroll/adopt/...) AWS resources, that were
+created manually (or by different means), into a CDK stack. Some resources can simply be
+deleted and recreated by CDK, but for others, this is not convenient: Typically stateful
+resources like S3 Buckets, DynamoDB tables, etc., cannot be easily deleted without an
+impact on the service.
+
+To import an existing resource to a CDK stack:
+
+* run a `cdk diff` to ensure there are no pending changes to the CDK stack you want to
+  import resources into - if there are, apply/discard them first
+* add corresponding constructs for the resources to be added in your stack - for example,
+  for an S3 bucket, add something like `new s3.Bucket(this, 'ImportedS3Bucket', {});` -
+  **no other changes must be done to the stack before the import is completed**
+* run `cdk deploy` with `--import-resources` argument to instruct CDK to start the import
+  operation
+* if resource definition contains all information needed for the import, this happens
+  automatically (e.g. an `s3.Bucket` construct has an explicit `bucketName` set),
+  otherwise, CDK will prompt user to provide neccessary identification information (e.g.
+  the bucket name)
+* after cdk deploy reports success, the resource is managed by CDK. Any subsequent
+  changes in the construct configuration will be reflected on the resource
+
+
 ### `cdk destroy`
 
 Deletes a stack from it's environment. This will cause the resources in the stack to be destroyed (unless they were

Diff for: packages/aws-cdk/bin/cdk.ts

@@ -124,7 +124,8 @@ async function parseCommandLineArguments() {
         desc: 'Continuously observe the project files, ' +
           'and deploy the given stack(s) automatically when changes are detected. ' +
           'Implies --hotswap by default',
-      }),
+      })
+      .option('import-resources', { type: 'boolean', alias: 'i', desc: 'Import existing resources in the stack' }),
     )
     .command('watch [STACKS..]', "Shortcut for 'deploy --watch'", yargs => yargs
       // I'm fairly certain none of these options, present for 'deploy', make sense for 'watch':
@@ -375,6 +376,7 @@ async function initCommandLine() {
           rollback: configuration.settings.get(['rollback']),
           hotswap: args.hotswap,
           watch: args.watch,
+          importResources: args['import-resources'],
         });
 
       case 'watch':

Diff for: packages/aws-cdk/lib/api/cloudformation-deployments.ts

@@ -6,7 +6,7 @@ import { publishAssets } from '../util/asset-publishing';
 import { Mode, SdkProvider } from './aws-auth';
 import { deployStack, DeployStackResult, destroyStack } from './deploy-stack';
 import { ToolkitInfo } from './toolkit-info';
-import { CloudFormationStack, Template } from './util/cloudformation';
+import { CloudFormationStack, Template, ResourcesToImport } from './util/cloudformation';
 import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor';
 
 /**
@@ -152,6 +152,11 @@ export interface DeployStackOptions {
    * @default - nothing extra is appended to the User-Agent header
    */
   readonly extraUserAgent?: string;
+
+  /**
+   * List of existing resources to be IMPORTED into the stack, instead of being CREATED
+   */
+  readonly resourcesToImport?: ResourcesToImport;
 }
 
 export interface DestroyStackOptions {
@@ -230,6 +235,7 @@ export class CloudFormationDeployments {
       rollback: options.rollback,
       hotswap: options.hotswap,
       extraUserAgent: options.extraUserAgent,
+      resourcesToImport: options.resourcesToImport,
     });
   }
 

Diff for: packages/aws-cdk/lib/api/deploy-stack.ts

@@ -14,7 +14,7 @@ import { CfnEvaluationException } from './hotswap/evaluate-cloudformation-templa
 import { ToolkitInfo } from './toolkit-info';
 import {
   changeSetHasNoChanges, CloudFormationStack, TemplateParameters, waitForChangeSet,
-  waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges,
+  waitForStackDeploy, waitForStackDelete, ParameterValues, ParameterChanges, ResourcesToImport,
 } from './util/cloudformation';
 import { StackActivityMonitor, StackActivityProgress } from './util/cloudformation/stack-activity-monitor';
 
@@ -189,6 +189,12 @@ export interface DeployStackOptions {
    * @default - nothing extra is appended to the User-Agent header
    */
   readonly extraUserAgent?: string;
+
+  /**
+   * If set, change set of type IMPORT will be created, and resourcesToImport
+   * passed to it.
+   */
+  readonly resourcesToImport?: ResourcesToImport;
 }
 
 const LARGE_TEMPLATE_SIZE_KB = 50;
@@ -294,7 +300,8 @@ async function prepareAndExecuteChangeSet(
   const changeSet = await cfn.createChangeSet({
     StackName: deployName,
     ChangeSetName: changeSetName,
-    ChangeSetType: update ? 'UPDATE' : 'CREATE',
+    ChangeSetType: options.resourcesToImport ? 'IMPORT' : update ? 'UPDATE' : 'CREATE',
+    ResourcesToImport: options.resourcesToImport,
     Description: `CDK Changeset for execution ${executionId}`,
     TemplateBody: bodyParameter.TemplateBody,
     TemplateURL: bodyParameter.TemplateURL,

Diff for: packages/aws-cdk/lib/api/util/cloudformation.ts

@@ -16,6 +16,8 @@ interface TemplateParameter {
   [key: string]: any;
 }
 
+export type ResourcesToImport = CloudFormation.ResourcesToImport;
+
 /**
  * Represents an (existing) Stack in CloudFormation
  *

Diff for: packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts

@@ -34,7 +34,7 @@ export class StackStatus {
   }
 
   get isDeploySuccess(): boolean {
-    return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE');
+    return !this.isNotFound && (this.name === 'CREATE_COMPLETE' || this.name === 'UPDATE_COMPLETE' || this.name === 'IMPORT_COMPLETE');
   }
 
   public toString(): string {

Diff for: packages/aws-cdk/lib/cdk-toolkit.ts

@@ -11,8 +11,10 @@ import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap';
 import { CloudFormationDeployments } from './api/cloudformation-deployments';
 import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly';
 import { CloudExecutable } from './api/cxapp/cloud-executable';
+import { ResourcesToImport } from './api/util/cloudformation';
 import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
 import { printSecurityDiff, printStackDiff, RequireApproval } from './diff';
+import { prepareResourcesToImport } from './import';
 import { data, debug, error, highlight, print, success, warning } from './logging';
 import { deserializeStructure } from './serialize';
 import { Configuration, PROJECT_CONFIG } from './settings';
@@ -117,6 +119,11 @@ export class CdkToolkit {
       return this.watch(options);
     }
 
+    // TODO - print more intelligent message
+    if (options.importResources) {
+      warning('Import resources flag was set');
+    }
+
     const stacks = await this.selectStacksForDeploy(options.selector, options.exclusively, options.cacheCloudAssembly);
 
     const requireApproval = options.requireApproval ?? RequireApproval.Broadening;
@@ -167,6 +174,17 @@ export class CdkToolkit {
         continue;
       }
 
+      let resourcesToImport: ResourcesToImport | undefined = undefined;
+      if (options.importResources) {
+        const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack);
+        resourcesToImport = await prepareResourcesToImport(currentTemplate, stack);
+
+        // There's a CloudFormation limitation that on import operation, no other changes are allowed:
+        // As CDK always changes the CDKMetadata resource with a new value, as a workaround, we override
+        // the template's metadata with currently deployed version
+        stack.template.Resources.CDKMetadata = currentTemplate.Resources.CDKMetadata;
+      }
+
       if (requireApproval !== RequireApproval.Never) {
         const currentTemplate = await this.props.cloudFormation.readCurrentTemplate(stack);
         if (printSecurityDiff(currentTemplate, stack, requireApproval)) {
@@ -209,6 +227,7 @@ export class CdkToolkit {
           rollback: options.rollback,
           hotswap: options.hotswap,
           extraUserAgent: options.extraUserAgent,
+          resourcesToImport,
         });
 
         const message = result.noOp
@@ -799,6 +818,14 @@ export interface DeployOptions extends WatchOptions {
    * @default true
    */
   readonly cacheCloudAssembly?: boolean;
+
+  /**
+   * Whether to import matching existing resources for newly defined constructs in the stack,
+   * rather than creating new ones
+   *
+   * @default false
+   */
+  readonly importResources?: boolean;
 }
 
 export interface DestroyOptions {

Diff for: packages/aws-cdk/lib/import.ts

@@ -0,0 +1,53 @@
+import * as cfnDiff from '@aws-cdk/cloudformation-diff';
+import * as cxapi from '@aws-cdk/cx-api';
+import * as promptly from 'promptly';
+import { ResourcesToImport } from './api/util/cloudformation';
+
+// Basic idea: we want to have a structure (ideally auto-generated from CFN definitions) that lists all resource types
+// that support importing and for each type, the identification information
+//
+// For each resource that is to be added in the new template:
+// - look up the identification information for the resource type [if not found, fail "type not supported"]
+// - look up the physical resource (perhaps using cloud control API?) [if not found, fail "resource to be imported does not exist"]
+// - assembe and return "resources to import" object to be passed on to changeset creation
+//
+// TEST: can we have a CFN changeset that both creates resources and import other resources?
+const RESOURCE_IDENTIFIERS: { [key: string]: string[] } = {
+  'AWS::S3::Bucket': ['BucketName'],
+};
+
+export async function prepareResourcesToImport(oldTemplate: any, newTemplate: cxapi.CloudFormationStackArtifact): Promise<ResourcesToImport> {
+  const diff = cfnDiff.diffTemplate(oldTemplate, newTemplate.template);
+
+  const additions: { [key: string]: cfnDiff.ResourceDifference } = {};
+  diff.resources.forEachDifference((id, chg) => {
+    if (chg.isAddition) {
+      additions[id] = chg;
+    }
+  });
+
+  const resourcesToImport: ResourcesToImport = [];
+  for (let [id, chg] of Object.entries(additions)) {
+    if (chg.newResourceType === undefined || !(chg.newResourceType in RESOURCE_IDENTIFIERS)) {
+      throw new Error(`Resource ${id} is of type ${chg.newResourceType} that is not supported for import`);
+    }
+
+    let identifier: { [key: string]: string } = {};
+    for (let idpart of RESOURCE_IDENTIFIERS[chg.newResourceType]) {
+      if (chg.newProperties && (idpart in chg.newProperties)) {
+        identifier[idpart] = chg.newProperties[idpart];
+      } else {
+        const displayName : string = newTemplate.template?.Resources?.[id]?.Metadata?.['aws:cdk:path'] ?? id;
+        identifier[idpart] = await promptly.prompt(`Please enter ${idpart} of ${chg.newResourceType} to import as ${displayName.replace(/\/Resource$/, '')}: `);
+      }
+    }
+
+    resourcesToImport.push({
+      LogicalResourceId: id,
+      ResourceType: chg.newResourceType,
+      ResourceIdentifier: identifier,
+    });
+  }
+
+  return resourcesToImport;
+}