Skip to content

Commit

Permalink
fix(integ-runner): cannot pass test name after --language (#28922)
Browse files Browse the repository at this point in the history 
Current situation:
yarn integ allows to run a TS integ file
CI doesn’t allow it, which caused confusing issues.
What we want instead:
Both local integ-runner and CI don’t allow it, so that this issue won’t happen again. To do this, we need to update integ-runner to disable running TS locally.

`integ-runner` package json should stay the same as it includes other tests in python and go.

Had a previous attempt in this PR #28770, but reverted as it caused an issue, see detail in #28891.

BREAKING CHANGE: Parsing of the cli input caused arguments passed after the first instance `--language <language>` to be interpreted as a language as well. This prevented passing a test name after providing cli options. To overcome this limitation, `integ-runner` now requires an explicit `--language` option for each language you want to include: `integ-runner --language javascript --language python`. This was already documented that way and always the intended way to use this feature.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
@GavinZZ @SankyRed
GavinZZ authored and SankyRed committed Feb 8, 2024
1 parent 8ad0d1a commit eaf8f53
Show file tree
Hide file tree
Showing 38 changed files with 54 additions and 38 deletions.
2 changes: 1 addition & 1 deletion packages/@aws-cdk-testing/framework-integ/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"lint": "cdk-lint",
"pkglint": "pkglint -f",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"package": "cdk-package",
"build+test": "yarn build && yarn test",
"build+extract": "yarn build",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/app-staging-synthesizer-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-amplify-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-appconfig-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-apprunner-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-cloud9-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-codestar-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-cognito-identitypool-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-gamelift-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-glue-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-iot-actions-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-iot-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-iotevents-actions-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-iotevents-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-ivs-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-kinesisanalytics-flink-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-kinesisfirehose-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-lambda-go-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-lambda-python-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-location-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-msk-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-neptune-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"pkglint": "pkglint -f",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-redshift-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-route53resolver-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"pkglint": "pkglint -f",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-s3objectlambda-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-sagemaker-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"pkglint": "pkglint -f",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-scheduler-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-scheduler-targets-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
"homepage": "https://github.com/aws/aws-cdk",
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-servicecatalogappregistry-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/cdk-cli-wrapper/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/custom-resource-handlers/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"version": "0.0.0",
"scripts": {
"build": "tsc -b && node scripts/generate.js",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/example-construct-library/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"watch": "cdk-watch",
"lint": "cdk-lint",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"pkglint": "pkglint -f",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
1 change: 1 addition & 0 deletions packages/@aws-cdk/integ-runner/lib/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ export function parseCliArgs(args: string[] = []) {
default: ['javascript', 'typescript', 'python', 'go'],
choices: ['javascript', 'typescript', 'python', 'go'],
type: 'array',
nargs: 1,
desc: 'Use these presets to run integration tests for the selected languages',
})
.option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/integ-tests-alpha/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"awslint": "cdk-awslint",
"pkglint": "pkglint -f",
"test": "cdk-test",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"watch": "cdk-watch",
"build+test": "yarn build && yarn test",
"build+test+package": "yarn build+test && yarn package",
Expand Down
2 changes: 1 addition & 1 deletion packages/cdk-assets/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
},
"scripts": {
"build": "cdk-build",
"integ": "integ-runner",
"integ": "integ-runner --language javascript",
"lint": "cdk-lint",
"package": "cdk-package",
"awslint": "cdk-awslint",
Expand Down
2 changes: 1 addition & 1 deletion tools/@aws-cdk/pkglint/lib/rules.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1231,7 +1231,7 @@ export class MustHaveIntegCommand extends ValidationRule {
public validate(pkg: PackageJson): void {
if (!hasIntegTests(pkg)) { return; }

expectJSON(this.name, pkg, 'scripts.integ', 'integ-runner');
expectJSON(this.name, pkg, 'scripts.integ', /integ-runner/, undefined, false, true);

// We can't ACTUALLY require cdk-build-tools/package.json here,
// because WE don't depend on cdk-build-tools and we don't know if
Expand Down
19 changes: 17 additions & 2 deletions tools/@aws-cdk/pkglint/lib/util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,32 @@ import { PackageJson, PKGLINT_IGNORES } from './packagejson';
/**
* Expect a particular JSON key to be a given value
*/
export function expectJSON(ruleName: string, pkg: PackageJson, jsonPath: string, expected: any, ignore?: RegExp, caseInsensitive: boolean = false) {
export function expectJSON(
ruleName: string,
pkg: PackageJson,
jsonPath: string,
expected: any,
ignore?: RegExp,
caseInsensitive: boolean = false,
regexMatch: boolean = false,
) {
const parts = jsonPath.split('.');
const actual = deepGet(pkg.json, parts);
if (applyCaseInsensitive(applyIgnore(actual)) !== applyCaseInsensitive(applyIgnore(expected))) {
if (checkEquality()) {
pkg.report({
ruleName,
message: `${jsonPath} should be ${JSON.stringify(expected)}${ignore ? ` (ignoring ${ignore})` : ''}, is ${JSON.stringify(actual)}`,
fix: () => { deepSet(pkg.json, parts, expected); },
});
}

function checkEquality(): boolean {
if (regexMatch) {
return !expected.test(applyCaseInsensitive(applyIgnore(actual)));
}
return applyCaseInsensitive(applyIgnore(actual)) !== applyCaseInsensitive(applyIgnore(expected));
}

function applyIgnore(val: any): string {
if (!ignore || val == null) { return JSON.stringify(val); }
const str = JSON.stringify(val);
Expand Down

0 comments on commit eaf8f53

Please sign in to comment.