feat(s3-deployment): deploy data with deploy-time values (aws#18659)

Allow deploying test-based content that can potentially include deploy-time values such as attributes of cloud resources.

Introduce a `Source.data(objectKey, text)` and `Source.jsonData(objectKey, obj)` where the data can naturally include deploy-time tokens such as references to resources (`Ref`) or to resource attributes (`Fn::GetAtt`).

For example:

```ts
const appConfig = {
  topic_arn: topic.topicArn,
  base_url: 'https://my-endpoint',
};

new s3deploy.BucketDeployment(this, 'BucketDeployment', {
  sources: [s3deploy.Source.jsonData('config.json', config)],
  destinationBucket: destinationBucket,
});
```

This is implemented by replacing the deploy-time tokens with markers that are replaced inside the s3-deployment custom resource.

Fixes aws#12903

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: Elad Ben-Israel

Diff for: packages/@aws-cdk/aws-s3-deployment/README.md

@@ -49,6 +49,10 @@ The following source types are supported for bucket deployments:
 - Local .zip file: `s3deploy.Source.asset('/path/to/local/file.zip')`
 - Local directory: `s3deploy.Source.asset('/path/to/local/directory')`
 - Another bucket: `s3deploy.Source.bucket(bucket, zipObjectKey)`
+- String data: `s3deploy.Source.data('object-key.txt', 'hello, world!')`
+  (supports [deploy-time values](#data-with-deploy-time-values))
+- JSON data: `s3deploy.Source.jsonData('object-key.json', { json: 'object' })`
+  (supports [deploy-time values](#data-with-deploy-time-values))
 
 To create a source from a single file, you can pass `AssetOptions` to exclude
 all but a single file:
@@ -268,6 +272,34 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
 });
 ```
 
+## Data with deploy-time values
+
+The content passed to `Source.data()` or `Source.jsonData()` can include
+references that will get resolved only during deployment.
+
+For example:
+
+```ts
+import * as sns from '@aws-cdk/aws-sns';
+
+declare const destinationBucket: s3.Bucket;
+declare const topic: sns.Topic;
+
+const appConfig = {
+  topic_arn: topic.topicArn,
+  base_url: 'https://my-endpoint',
+};
+
+new s3deploy.BucketDeployment(this, 'BucketDeployment', {
+  sources: [s3deploy.Source.jsonData('config.json', appConfig)],
+  destinationBucket,
+});
+```
+
+The value in `topic.topicArn` is a deploy-time value. It only gets resolved
+during deployment by placing a marker in the generated source file and
+substituting it when its deployed to the destination with the actual value.
+
 ## Notes
 
 - This library uses an AWS CloudFormation custom resource which about 10MiB in
@@ -282,7 +314,7 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
   be good enough: the custom resource will simply not run if the properties don't
   change.
   - If you use assets (`s3deploy.Source.asset()`) you don't need to worry
-    about this: the asset system will make sure that if the files have changed, 
+    about this: the asset system will make sure that if the files have changed,
     the file name is unique and the deployment will run.
 
 ## Development

Diff for: packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts

@@ -323,13 +323,18 @@ export class BucketDeployment extends CoreConstruct {
       }));
     }
 
+    // to avoid redundant stack updates, only include "SourceMarkers" if one of
+    // the sources actually has markers.
+    const hasMarkers = sources.some(source => source.markers);
+
     const crUniqueId = `CustomResource${this.renderUniqueId(props.memoryLimit, props.vpc)}`;
     const cr = new cdk.CustomResource(this, crUniqueId, {
       serviceToken: handler.functionArn,
       resourceType: 'Custom::CDKBucketDeployment',
       properties: {
         SourceBucketNames: sources.map(source => source.bucket.bucketName),
         SourceObjectKeys: sources.map(source => source.zipObjectKey),
+        SourceMarkers: hasMarkers ? sources.map(source => source.markers ?? {}) : undefined,
         DestinationBucketName: props.destinationBucket.bucketName,
         DestinationBucketKeyPrefix: props.destinationKeyPrefix,
         RetainOnDelete: props.retainOnDelete,

Diff for: packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py

@@ -20,6 +20,7 @@
 CFN_SUCCESS = "SUCCESS"
 CFN_FAILED = "FAILED"
 ENV_KEY_MOUNT_PATH = "MOUNT_PATH"
+ENV_KEY_SKIP_CLEANUP = "SKIP_CLEANUP"
 
 CUSTOM_RESOURCE_OWNER_TAG = "aws-cdk:cr-owned"
 
@@ -45,6 +46,7 @@ def cfn_error(message=None):
         try:
             source_bucket_names = props['SourceBucketNames']
             source_object_keys  = props['SourceObjectKeys']
+            source_markers      = props.get('SourceMarkers', None)
             dest_bucket_name    = props['DestinationBucketName']
             dest_bucket_prefix  = props.get('DestinationBucketKeyPrefix', '')
             retain_on_delete    = props.get('RetainOnDelete', "true") == "true"
@@ -55,6 +57,11 @@ def cfn_error(message=None):
             exclude             = props.get('Exclude', [])
             include             = props.get('Include', [])
 
+            # backwards compatibility - if "SourceMarkers" is not specified,
+            # assume all sources have an empty market map
+            if source_markers is None:
+                source_markers = [{} for i in range(len(source_bucket_names))]
+
             default_distribution_path = dest_bucket_prefix
             if not default_distribution_path.endswith("/"):
                 default_distribution_path += "/"
@@ -71,7 +78,7 @@ def cfn_error(message=None):
         if dest_bucket_prefix == "/":
             dest_bucket_prefix = ""
 
-        s3_source_zips = map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys)
+        s3_source_zips = list(map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys))
         s3_dest = "s3://%s/%s" % (dest_bucket_name, dest_bucket_prefix)
         old_s3_dest = "s3://%s/%s" % (old_props.get("DestinationBucketName", ""), old_props.get("DestinationBucketKeyPrefix", ""))
 
@@ -106,7 +113,7 @@ def cfn_error(message=None):
             aws_command("s3", "rm", old_s3_dest, "--recursive")
 
         if request_type == "Update" or request_type == "Create":
-            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include)
+            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
 
         if distribution_id:
             cloudfront_invalidate(distribution_id, distribution_paths)
@@ -120,7 +127,11 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
+    # list lengths are equal
+    if len(s3_source_zips) != len(source_markers):
+        raise Exception("'source_markers' and 's3_source_zips' must be the same length")
+
     # create a temporary working directory in /tmp or if enabled an attached efs volume
     if ENV_KEY_MOUNT_PATH in os.environ:
         workdir = os.getenv(ENV_KEY_MOUNT_PATH) + "/" + str(uuid4())
@@ -136,13 +147,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
 
     try:
         # download the archive from the source and extract to "contents"
-        for s3_source_zip in s3_source_zips:
+        for i in range(len(s3_source_zips)):
+            s3_source_zip = s3_source_zips[i]
+            markers       = source_markers[i]
+
             archive=os.path.join(workdir, str(uuid4()))
             logger.info("archive: %s" % archive)
             aws_command("s3", "cp", s3_source_zip, archive)
             logger.info("| extracting archive to: %s\n" % contents_dir)
-            with ZipFile(archive, "r") as zip:
-              zip.extractall(contents_dir)
+            logger.info("| markers: %s" % markers)
+            extract_and_replace_markers(archive, contents_dir, markers)
 
         # sync from "contents" to destination
 
@@ -163,7 +177,8 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
         s3_command.extend(create_metadata_args(user_metadata, system_metadata))
         aws_command(*s3_command)
     finally:
-        shutil.rmtree(workdir)
+        if not os.getenv(ENV_KEY_SKIP_CLEANUP):
+            shutil.rmtree(workdir)
 
 #---------------------------------------------------------------------------------------------------
 # invalidate files in the CloudFront distribution edge caches
@@ -257,3 +272,29 @@ def bucket_owned(bucketName, keyPrefix):
         logger.info("| error getting tags from bucket")
         logger.exception(e)
         return False
+
+# extract archive and replace markers in output files
+def extract_and_replace_markers(archive, contents_dir, markers):
+    with ZipFile(archive, "r") as zip:
+        zip.extractall(contents_dir)
+
+        # replace markers for this source
+        for file in zip.namelist():
+            file_path = os.path.join(contents_dir, file)
+            if os.path.isdir(file_path): continue
+            replace_markers(file_path, markers)    
+
+def replace_markers(filename, markers):
+    # convert the dict of string markers to binary markers
+    replace_tokens = dict([(k.encode('utf-8'), v.encode('utf-8')) for k, v in markers.items()])
+
+    outfile = filename + '.new'
+    with open(filename, 'rb') as fi, open(outfile, 'wb') as fo:
+        for line in fi:
+            for token in replace_tokens:
+                line = line.replace(token, replace_tokens[token])
+            fo.write(line)
+
+    # # delete the original file and rename the new one to the original
+    os.remove(filename)
+    os.rename(outfile, filename)

Diff for: packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts

@@ -0,0 +1,81 @@
+import { Stack } from '@aws-cdk/core';
+
+// keep this import separate from other imports to reduce chance for merge conflicts with v2-main
+// eslint-disable-next-line no-duplicate-imports, import/order
+import { Construct } from '@aws-cdk/core';
+
+export interface Content {
+  readonly text: string;
+  readonly markers: Record<string, any>;
+}
+
+/**
+ * Renders the given string data as deployable content with markers substituted
+ * for all "Ref" and "Fn::GetAtt" objects.
+ *
+ * @param scope Construct scope
+ * @param data The input data
+ * @returns The markered text (`text`) and a map that maps marker names to their
+ * values (`markers`).
+ */
+export function renderData(scope: Construct, data: string): Content {
+  const obj = Stack.of(scope).resolve(data);
+  if (typeof(obj) === 'string') {
+    return { text: obj, markers: {} };
+  }
+
+  if (typeof(obj) !== 'object') {
+    throw new Error(`Unexpected: after resolve() data must either be a string or a CloudFormation intrinsic. Got: ${JSON.stringify(obj)}`);
+  }
+
+  let markerIndex = 0;
+  const markers: Record<string, FnJoinPart> = {};
+  const result = new Array<string>();
+  const fnJoin: FnJoin | undefined = obj['Fn::Join'];
+
+  if (fnJoin) {
+    const sep = fnJoin[0];
+    const parts = fnJoin[1];
+
+    if (sep !== '') {
+      throw new Error(`Unexpected "Fn::Join", expecting separator to be an empty string but got "${sep}"`);
+    }
+
+    for (const part of parts) {
+      if (typeof (part) === 'string') {
+        result.push(part);
+        continue;
+      }
+
+      if (typeof (part) === 'object') {
+        addMarker(part);
+        continue;
+      }
+
+      throw new Error(`Unexpected "Fn::Join" part, expecting string or object but got ${typeof (part)}`);
+    }
+
+  } else if (obj.Ref || obj['Fn::GetAtt']) {
+    addMarker(obj);
+  } else {
+    throw new Error('Unexpected: Expecting `resolve()` to return "Fn::Join", "Ref" or "Fn::GetAtt"');
+  }
+
+  function addMarker(part: Ref | GetAtt) {
+    const keys = Object.keys(part);
+    if (keys.length !== 1 || (keys[0] != 'Ref' && keys[0] != 'Fn::GetAtt')) {
+      throw new Error(`Invalid CloudFormation reference. "Ref" or "Fn::GetAtt". Got ${JSON.stringify(part)}`);
+    }
+
+    const marker = `<<marker:0xbaba:${markerIndex++}>>`;
+    result.push(marker);
+    markers[marker] = part;
+  }
+
+  return { text: result.join(''), markers };
+}
+
+type FnJoin = [string, FnJoinPart[]];
+type FnJoinPart = string | Ref | GetAtt;
+type Ref = { Ref: string };
+type GetAtt = { 'Fn::GetAtt': [string, string] };

Diff for: packages/@aws-cdk/aws-s3-deployment/lib/source.ts

@@ -1,6 +1,10 @@
+import * as fs from 'fs';
+import { join, dirname } from 'path';
 import * as iam from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
 import * as s3_assets from '@aws-cdk/aws-s3-assets';
+import { FileSystem, Stack } from '@aws-cdk/core';
+import { renderData } from './render-data';
 
 // keep this import separate from other imports to reduce chance for merge conflicts with v2-main
 // eslint-disable-next-line no-duplicate-imports, import/order
@@ -19,6 +23,12 @@ export interface SourceConfig {
    * An S3 object key in the source bucket that points to a zip file.
    */
   readonly zipObjectKey: string;
+
+  /**
+   * A set of markers to substitute in the source content.
+   * @default - no markers
+   */
+  readonly markers?: Record<string, any>;
 }
 
 /**
@@ -50,6 +60,8 @@ export interface ISource {
  *     Source.bucket(bucket, key)
  *     Source.asset('/local/path/to/directory')
  *     Source.asset('/local/path/to/a/file.zip')
+ *     Source.data('hello/world/file.txt', 'Hello, world!')
+ *     Source.data('config.json', { baz: topic.topicArn })
  *
  */
 export class Source {
@@ -110,5 +122,51 @@ export class Source {
     };
   }
 
+  /**
+   * Deploys an object with the specified string contents into the bucket. The
+   * content can include deploy-time values (such as `snsTopic.topicArn`) that
+   * will get resolved only during deployment.
+   *
+   * To store a JSON object use `Source.jsonData()`.
+   *
+   * @param objectKey The destination S3 object key (relative to the root of the
+   * S3 deployment).
+   * @param data The data to be stored in the object.
+   */
+  public static data(objectKey: string, data: string): ISource {
+    return {
+      bind: (scope: Construct, context?: DeploymentSourceContext) => {
+        const workdir = FileSystem.mkdtemp('s3-deployment');
+        const outputPath = join(workdir, objectKey);
+        const rendered = renderData(scope, data);
+        fs.mkdirSync(dirname(outputPath), { recursive: true });
+        fs.writeFileSync(outputPath, rendered.text);
+        const asset = this.asset(workdir).bind(scope, context);
+        return {
+          bucket: asset.bucket,
+          zipObjectKey: asset.zipObjectKey,
+          markers: rendered.markers,
+        };
+      },
+    };
+  }
+
+  /**
+   * Deploys an object with the specified JSON object into the bucket. The
+   * object can include deploy-time values (such as `snsTopic.topicArn`) that
+   * will get resolved only during deployment.
+   *
+   * @param objectKey The destination S3 object key (relative to the root of the
+   * S3 deployment).
+   * @param obj A JSON object.
+   */
+  public static jsonData(objectKey: string, obj: any): ISource {
+    return {
+      bind: (scope: Construct, context?: DeploymentSourceContext) => {
+        return Source.data(objectKey, Stack.of(scope).toJsonString(obj)).bind(scope, context);
+      },
+    };
+  }
+
   private constructor() { }
 }