feat(cli): regression tests

packages/aws-cdk/.gitignore

@@ -17,3 +17,7 @@ nyc.config.js
 
 !test/integ/run-wrappers/dist
 !test/integ/cli/**/*
+
+# Might be created when running regression tests.
+# But normally should be deleted after execution.
+test/integ/cli-backwards-tests-*

packages/aws-cdk/CONTRIBUTING.md

@@ -16,7 +16,41 @@ REQUIREMENTS
 Run:
 
 ```
-npm run integ-cli
+yarn integ-cli
+```
+
+This command runs two types of tests:
+
+#### Functional
+
+These tests simply run the local integration tests located in [`test/integ/cli`](./test/integ/cli). They test the proper deployment of stacks and in general the correctness of the actions performed by the CLI.
+
+You can also run just these tests by executing:
+
+```console
+yarn integ-cli-no-regression
+```
+
+#### Regression
+
+Validate that previously tested functionality still works in light of recent changes to the CLI. This is done by fetching the functional tests of the latest published release, and running them against the new CLI code.
+
+These tests run in two variations:
+
+- **against local framework code**
+
+  Use your local framework code. This is important to make sure the new CLI version
+  will work properly with the new framework version. 
+
+- **against latest release code**
+
+  Fetches the framework code from the latest release. This is important to make sure
+  the new CLI version does not rely on new framework features to provide the same functionality.
+
+You can also run just these tests by executing:
+
+```console
+yarn integ-cli-regression
 ```
 
 ### Init template integration tests

packages/aws-cdk/package.json

@@ -17,7 +17,9 @@
     "package": "cdk-package",
     "build+test+package": "npm run build+test && npm run package",
     "build+test": "npm run build && npm test",
-    "integ-cli": "test/integ/run-against-repo test/integ/cli/test.sh",
+    "integ-cli": "scripts/integ-cli.sh",
+    "integ-cli-regression": "scripts/integ-cli-regression.sh",
+    "integ-cli-no-regression": "scripts/integ-cli-no-regression.sh",
     "integ-init": "test/integ/run-against-dist test/integ/init/test-all.sh"
   },
   "cdk-build": {

packages/aws-cdk/scripts/integ-cli-no-regression.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# 
+# Run only local integration tests.
+#
+set -eu
+
+scriptdir=$(cd $(dirname $0) && pwd)
+
+echo "Running local integration tests"
+./test/integ/run-against-repo test/integ/cli/test.sh

packages/aws-cdk/scripts/integ-cli-regression.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# 
+# Run only local integration tests.
+#
+set -eu
+
+scriptdir=$(cd $(dirname $0) && pwd)
+repo_root=$(realpath ${scriptdir}/..)
+
+echo "Running regression tests against local code"
+${repo_root}/test/integ/run-against-repo ${repo_root}/test/integ/test-cli-regression-against-current-code.sh
+
+echo "Running regression tests against local CLI and published framework"
+${repo_root}/test/integ/run-against-repo ${repo_root}/test/integ/test-cli-regression-against-latest-release.sh

packages/aws-cdk/scripts/integ-cli.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# 
+# Run both local integration tests and regression tests.
+#
+set -eu
+
+scriptdir=$(cd $(dirname $0) && pwd)
+
+${scriptdir}/integ-cli-no-regression.sh
+${scriptdir}/integ-cli-regression.sh

packages/aws-cdk/test/integ/cli.exclusions.js

@@ -0,0 +1,54 @@
+/**
+List of exclusions when running backwards compatibility tests.
+Add when you need to exclude a specific integration test from a specific version.
+
+This is an escape hatch for the rare cases where we need to introduce
+a change that breaks existing integration tests. (e.g security)
+
+For example:
+
+{
+    "test": "test-cdk-iam-diff.sh",
+    "version": "1.30.0",
+    "justification": "iam policy generation has changed in version > 1.30.0 because..."
+},
+
+*/
+const exclusions = []
+
+function getExclusion(test, version) {
+
+    const filtered = exclusions.filter(e => {
+        return e.test === test && e.version === version;
+    });
+
+    if (filtered.length === 0) {
+        return undefined;
+    }
+
+    if (filtered.length === 1) {
+        return filtered[0];
+    }
+
+    throw new Error(`Multiple exclusions found for (${test, version}): ${filtered.length}`);
+
+}
+
+module.exports.shouldSkip = function (test, version) {
+
+    const exclusion = getExclusion(test, version);
+
+    return exclusion != undefined
+
+}
+
+module.exports.getJustification = function (test, version) {
+
+    const exclusion = getExclusion(test, version);
+
+    if (!exclusion) {
+        throw new Error(`Exclusion not found for (${test}, ${version})`);
+    }
+
+    return exclusion.justification;
+}

packages/aws-cdk/test/integ/cli/common.bash

@@ -72,14 +72,14 @@ function prepare_fixture() {
     mkdir -p node_modules
 
     npm install \
-        @aws-cdk/core@^1 \
-        @aws-cdk/aws-sns@^1 \
-        @aws-cdk/aws-iam@^1 \
-        @aws-cdk/aws-lambda@^1 \
-        @aws-cdk/aws-ssm@^1 \
-        @aws-cdk/aws-ecr-assets@^1 \
-        @aws-cdk/aws-cloudformation@^1 \
-        @aws-cdk/aws-ec2@^1
+        @aws-cdk/core \
+        @aws-cdk/aws-sns \
+        @aws-cdk/aws-iam \
+        @aws-cdk/aws-lambda \
+        @aws-cdk/aws-ssm \
+        @aws-cdk/aws-ecr-assets \
+        @aws-cdk/aws-cloudformation \
+        @aws-cdk/aws-ec2
 
     echo "| setup complete at: $PWD"
     echo "| 'cdk' is: $(type -p cdk)"

packages/aws-cdk/test/integ/cli/test-cdk-deploy-nested-stack-with-parameters.sh

@@ -6,7 +6,9 @@ source ${scriptdir}/common.bash
 
 setup
 
-stack_arn=$(cdk deploy -v ${STACK_NAME_PREFIX}-with-nested-stack-using-parameters --parameters "MyTopicParam=ThereIsNoSpoon")
+# STACK_NAME_PREFIX is used in MyTopicParam to allow multiple instances
+# of this test to run in parallel, othewise they will attempt to create the same SNS topic.
+stack_arn=$(cdk deploy -v ${STACK_NAME_PREFIX}-with-nested-stack-using-parameters --parameters "MyTopicParam=${STACK_NAME_PREFIX}ThereIsNoSpoon")
 echo "Stack deployed successfully"
 
 # verify that we only deployed a single stack (there's a single ARN in the output)

packages/aws-cdk/test/integ/cli/test-cdk-deploy-wildcard-with-parameters.sh

@@ -6,7 +6,9 @@ source ${scriptdir}/common.bash
 
 setup
 
-stack_arns=$(cdk deploy ${STACK_NAME_PREFIX}-param-test-\* --parameters "${STACK_NAME_PREFIX}-param-test-1:TopicNameParam=bazinga" --parameters "${STACK_NAME_PREFIX}-param-test-2:OtherTopicNameParam=ThatsMySpot")
+# STACK_NAME_PREFIX is used in OtherTopicNameParam to allow multiple instances
+# of this test to run in parallel, othewise they will attempt to create the same SNS topic.
+stack_arns=$(cdk deploy ${STACK_NAME_PREFIX}-param-test-\* --parameters "${STACK_NAME_PREFIX}-param-test-1:TopicNameParam=${STACK_NAME_PREFIX}bazinga" --parameters "${STACK_NAME_PREFIX}-param-test-2:OtherTopicNameParam=${STACK_NAME_PREFIX}ThatsMySpot")
 echo "Stack deployed successfully"
 
 # verify that we only deployed a single stack (there's a single ARN in the output)

packages/aws-cdk/test/integ/cli/test-cdk-deploy-with-parameters.sh

@@ -6,7 +6,9 @@ source ${scriptdir}/common.bash
 
 setup
 
-stack_arn=$(cdk deploy -v ${STACK_NAME_PREFIX}-param-test-1 --parameters "TopicNameParam=bazinga")
+# STACK_NAME_PREFIX is used in TopicNameParam to allow multiple instances
+# of this test to run in parallel, othewise they will attempt to create the same SNS topic.
+stack_arn=$(cdk deploy -v ${STACK_NAME_PREFIX}-param-test-1 --parameters "TopicNameParam=${STACK_NAME_PREFIX}bazinga")
 echo "Stack deployed successfully"
 
 # verify that we only deployed a single stack (there's a single ARN in the output)

packages/aws-cdk/test/integ/cli/test.sh

@@ -7,9 +7,46 @@ header CLI Integration Tests
 
 prepare_fixture
 
+current_version=$(node -p "require('${scriptdir}/../../../package.json').version")
+
+# This allows injecting different versions, not just the current one.
+# Useful when testing.
+VERSION_UNDER_TEST=${VERSION_UNDER_TEST:-${current_version}}
+
+# check if a specific test should be skiped
+# from execution in the current version.
+function should_skip {
+  test=$1
+  echo $(node -p "require('${scriptdir}/../cli.exclusions.js').shouldSkip('${test}', '${VERSION_UNDER_TEST}')")
+}
+
+# get the justification for why a test is skipped.
+# this will fail if there is no justification!
+function get_skip_jusitification {
+  test=$1
+  echo $(node -p "require('${scriptdir}/../cli.exclusions.js').getJustification('${test}', '${VERSION_UNDER_TEST}')")
+}
+
 for test in $(cd ${scriptdir} && ls test-*.sh); do
   echo "============================================================================================"
+
+  # first check this if this test should be skipped.
+  # this can happen when running in regression mode 
+  # when we introduce an intentional breaking change.
+  skip=$(should_skip ${test})
+
+  if [ ${skip} == "true" ]; then
+
+    # make sure we have a justification, this will fail if not.
+    jusitification="$(get_skip_jusitification ${test})"
+
+    # skip this specific test.
+    echo "${test} - skipped (${jusitification})"
+    continue
+  fi
+
   echo "${test}"
   echo "============================================================================================"
-  /bin/bash ${scriptdir}/${test}
+  # /bin/bash ${scriptdir}/${test}
 done
+

packages/aws-cdk/test/integ/run-against-dist.bash

@@ -22,17 +22,53 @@ function serve_npm_packages() {
     return
   fi
 
-  tarballs=$dist_root/js/*.tgz
+  if [ ! -z "${USE_PUBLISHED_FRAMEWORK_VERSION:-}" ]; then
 
-  log "Discovering local package names..."
-  # Read the package names from each tarball, so that we can generate
-  # a Verdaccio config that will keep each of these packages locally
-  # and not go to NPMJS for it.
-  package_names=""
-  for tgz in $tarballs; do
-    name=$(node -pe 'JSON.parse(process.argv[1]).name' "$(tar xOzf $tgz package/package.json)")
-    package_names="$package_names $name"
-  done
+    echo "Testing against latest published versions of the framework"
+
+    # when using latest published framework, only
+    # install the cli and its dependencies.
+
+    cli_root=./package  
+
+    version=$(node -p "require('${cli_root}/package.json').version")
+
+    # good lord
+    echo "Fetching CLI dependencies"
+    cli_deps=$(node -p "Object.entries(require('${cli_root}/package.json').dependencies).filter(x => x[0].includes('@aws-cdk')).map(x => x[0].replace('@aws-cdk/', '')).join(' ')")
+
+    tarballs=$dist_root/js/aws-cdk-${version}.tgz
+    package_names="aws-cdk"
+
+    for dep in ${cli_deps}; do
+      tarball=$dist_root/js/${dep}@${version}.jsii.tgz
+      package=@aws-cdk/${dep}
+      if [ ! -f ${tarball} ]; then
+        # not a jsii dependency, the tarball is different...
+        tarball=$dist_root/js/aws-cdk-${dep}-${version}.tgz        
+      fi 
+
+      tarballs="${tarballs} ${tarball}"
+      package_names="${package_names} ${package}"
+    done
+
+  else
+
+    echo "Testing against local versions of the framework"
+
+    tarballs=$dist_root/js/*.tgz
+
+    log "Discovering local package names..."
+    # Read the package names from each tarball, so that we can generate
+    # a Verdaccio config that will keep each of these packages locally
+    # and not go to NPMJS for it.
+    package_names=""
+    for tgz in $tarballs; do
+      name=$(node -pe 'JSON.parse(process.argv[1]).name' "$(tar xOzf $tgz package/package.json)")
+      package_names="$package_names $name"
+    done
+
+  fi
 
   #------------------------------------------------------------------------------
   # Start a local npm repository and install the CDK from the distribution to it
@@ -65,6 +101,7 @@ function serve_npm_packages() {
     # aws-cdk package directory.
     (cd $npmws && npm --quiet publish $tgz)
   done
+
 }
 
 function write_verdaccio_config() {

packages/aws-cdk/test/integ/run-wrappers/repo/npm

@@ -5,7 +5,11 @@ command=$1
 
 lerna=$REPO_ROOT/node_modules/.bin/lerna
 
-if [[ "$command" == "install" || "$command" == "i" ]]; then
+if [ ! -z "${USE_PUBLISHED_FRAMEWORK_VERSION:-}" ]; then
+  # simply pass through to npm and install the published package.
+  echo "Running original NPM command since we are testing against published framework version"
+  exec $ORIGINAL_NPM "$@"
+elif [[ "$command" == "install" || "$command" == "i" ]]; then
   npmargs="install"
   shift
 
@@ -29,5 +33,3 @@ if [[ "$command" == "install" || "$command" == "i" ]]; then
 
   exec $ORIGINAL_NPM $npmargs
 fi
-
-exec $ORIGINAL_NPM "$@"