diff --git a/.projenrc.ts b/.projenrc.ts index d6c9d7e650..6580f01169 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -136,6 +136,7 @@ const serviceSpecSchemaTask = serviceSpecImporters.addTask('gen-schemas', { 'SamTemplateSchema', 'CloudWatchConsoleServiceDirectory', 'GetAttAllowList', + 'CfnPrimaryIdentifierOverrides', 'OobRelationshipData', ].map((typeName: string) => ({ exec: [ diff --git a/packages/@aws-cdk/aws-service-spec/build/full-database.ts b/packages/@aws-cdk/aws-service-spec/build/full-database.ts index 75ef254fa6..5c1e1af203 100644 --- a/packages/@aws-cdk/aws-service-spec/build/full-database.ts +++ b/packages/@aws-cdk/aws-service-spec/build/full-database.ts @@ -30,7 +30,8 @@ export class FullDatabase extends DatabaseBuilder { .importLogSources(path.join(SOURCES, 'LogSources/log-source-resource.json')) .importScrutinies() .importAugmentations() - .importEventBridgeSchema(path.join(SOURCES, 'EventBridgeSchema')); + .importEventBridgeSchema(path.join(SOURCES, 'EventBridgeSchema')) + .importCfnPrimaryIdentifierOverrides(path.join(SOURCES, 'CloudFormationRefOverrides/ref-overrides.json')); } /** diff --git a/packages/@aws-cdk/service-spec-importers/.projen/tasks.json b/packages/@aws-cdk/service-spec-importers/.projen/tasks.json index ff9a202358..8dd8e35f97 100644 --- a/packages/@aws-cdk/service-spec-importers/.projen/tasks.json +++ b/packages/@aws-cdk/service-spec-importers/.projen/tasks.json @@ -154,6 +154,9 @@ { "exec": "ts-json-schema-generator --tsconfig tsconfig.json --type GetAttAllowList --out schemas/GetAttAllowList.schema.json" }, + { + "exec": "ts-json-schema-generator --tsconfig tsconfig.json --type CfnPrimaryIdentifierOverrides --out schemas/CfnPrimaryIdentifierOverrides.schema.json" + }, { "exec": "ts-json-schema-generator --tsconfig tsconfig.json --type OobRelationshipData --out schemas/OobRelationshipData.schema.json" } diff --git a/packages/@aws-cdk/service-spec-importers/schemas/CfnPrimaryIdentifierOverrides.schema.json b/packages/@aws-cdk/service-spec-importers/schemas/CfnPrimaryIdentifierOverrides.schema.json new file mode 100644 index 0000000000..7f1a1518c7 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/schemas/CfnPrimaryIdentifierOverrides.schema.json @@ -0,0 +1,16 @@ +{ + "$ref": "#/definitions/CfnPrimaryIdentifierOverrides", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "CfnPrimaryIdentifierOverrides": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "description": "Maps a resource type to a list of properties that are overrides of the CCAPI schema primary identifiers", + "type": "object" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/service-spec-importers/src/db-builder.ts b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts index 3a6fd67c61..a1238d21e7 100644 --- a/packages/@aws-cdk/service-spec-importers/src/db-builder.ts +++ b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts @@ -3,6 +3,7 @@ import { emptyDatabase, SpecDatabase } from '@aws-cdk/service-spec-types'; import { assertSuccess, Result } from '@cdklabs/tskb'; import { importArnTemplates } from './importers/import-arn-templates'; import { importCannedMetrics } from './importers/import-canned-metrics'; +import { importCfnPrimaryIdentifierOverrides } from './importers/import-cfn-primaryidentifier-overrides'; import { importCloudFormationDocumentation } from './importers/import-cloudformation-docs'; import { importCloudFormationRegistryResource } from './importers/import-cloudformation-registry'; import { importEventBridgeSchema } from './importers/import-eventbridge-schema'; @@ -23,6 +24,7 @@ import { loadSamSchema, loadSamSpec, loadOobRelationships, + loadCfnPrimaryIdentifierOverrides, } from './loaders'; import { loadDefaultEventBridgeSchema } from './loaders/load-eventbridge-schema'; import { JsonLensPatcher } from './patching'; @@ -198,6 +200,20 @@ export class DatabaseBuilder { }); } + /** + * Import patches to primary identifiers + */ + public importCfnPrimaryIdentifierOverrides(specFilePath: string) { + return this.addSourceImporter(async (db, report) => { + const cfnIdentifiers = this.loadResult( + await loadCfnPrimaryIdentifierOverrides(specFilePath, this.options), + report, + ); + + importCfnPrimaryIdentifierOverrides(db, cfnIdentifiers); + }); + } + public importArnTemplates(filePath: string) { return this.addSourceImporter(async (db, report) => { const arnFormatIndex = JSON.parse(await fs.readFile(filePath, { encoding: 'utf-8' })); diff --git a/packages/@aws-cdk/service-spec-importers/src/db-diff.ts b/packages/@aws-cdk/service-spec-importers/src/db-diff.ts index 6069e5f07b..e50e84d6cf 100644 --- a/packages/@aws-cdk/service-spec-importers/src/db-diff.ts +++ b/packages/@aws-cdk/service-spec-importers/src/db-diff.ts @@ -117,9 +117,8 @@ export class DbDiff { vendedLogs: diffField(a, b, 'vendedLogs', jsonEq), vendedLogsConfig: diffField(a, b, 'vendedLogsConfig', jsonEq), tagInformation: diffField(a, b, 'tagInformation', jsonEq), - primaryIdentifier: collapseEmptyDiff( - diffList(a.primaryIdentifier ?? [], b.primaryIdentifier ?? [], (x, y) => x === y), - ), + primaryIdentifier: diffField(a, b, 'primaryIdentifier', jsonEq), + cfnRefIdentifier: diffField(a, b, 'cfnRefIdentifier', jsonEq), attributes: collapseEmptyDiff(diffMap(a.attributes, b.attributes, (x, y) => this.diffAttribute(x, y))), properties: collapseEmptyDiff(diffMap(a.properties, b.properties, (x, y) => this.diffProperty(x, y))), typeDefinitionDiff: this.diffResourceTypeDefinitions(a, b), diff --git a/packages/@aws-cdk/service-spec-importers/src/diff-fmt.ts b/packages/@aws-cdk/service-spec-importers/src/diff-fmt.ts index 182b7adae2..4303ccc5f6 100644 --- a/packages/@aws-cdk/service-spec-importers/src/diff-fmt.ts +++ b/packages/@aws-cdk/service-spec-importers/src/diff-fmt.ts @@ -115,6 +115,8 @@ export class DiffFormatter { 'tagInformation', 'arnTemplate', 'vendedLogs', + 'primaryIdentifier', + 'cfnRefIdentifier', ]), ).indent(META_INDENT), listWithCaption('properties', this.renderProperties(r.properties, db)), @@ -150,6 +152,9 @@ export class DiffFormatter { 'scrutinizable', 'tagInformation', 'arnTemplate', + 'vendedLogs', + 'primaryIdentifier', + 'cfnRefIdentifier', ]); return new PrintableTree(`resource ${key}`).addBullets([ diff --git a/packages/@aws-cdk/service-spec-importers/src/importers/eventbridge/event-resource-matcher.ts b/packages/@aws-cdk/service-spec-importers/src/importers/eventbridge/event-resource-matcher.ts index 4eef668597..d152dcd4a4 100644 --- a/packages/@aws-cdk/service-spec-importers/src/importers/eventbridge/event-resource-matcher.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/eventbridge/event-resource-matcher.ts @@ -67,8 +67,6 @@ function eventDecider({ const resourceMatches = matchTypeFieldsToResources(resources, typeInfos); if (resourceMatches.length > 0) { - // TODO: remove this - console.log(`Resources Matches = ${resourceMatches.length}`); return { resource: resourceMatches[0].resource, matches: resourceMatches[0].matches }; } else if (resourceMatches.length == 0) { // TODO: change this to report @@ -112,7 +110,7 @@ function matchTypeFieldsToResources(resources: Resource[], typeInfos: EventTypeD if (matches.length > 0) { if (matches.length > 1) { //TODO: 17 events affected by this, some of them has resourceId & resourceName, some has in multiple levels the resource - console.log('here we are', { resource, matches: JSON.stringify(matches, null, 2) }); + // console.log('here we are', { resource, matches: JSON.stringify(matches, null, 2) }); } resourceMatches.push({ resource, diff --git a/packages/@aws-cdk/service-spec-importers/src/importers/import-cfn-primaryidentifier-overrides.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-cfn-primaryidentifier-overrides.ts new file mode 100644 index 0000000000..1a508fe15c --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-cfn-primaryidentifier-overrides.ts @@ -0,0 +1,29 @@ +import { SpecDatabase } from '@aws-cdk/service-spec-types'; +import { CfnPrimaryIdentifierOverrides } from '../types'; + +/** + * For the given resources, update the primary identifiers of these resources + */ +export function importCfnPrimaryIdentifierOverrides(db: SpecDatabase, allowList: CfnPrimaryIdentifierOverrides) { + const errors = new Array(); + + for (const [resourceType, propNames] of Object.entries(allowList)) { + try { + const resource = db.lookup('resource', 'cloudFormationType', 'equals', resourceType).only(); + + for (const propName of propNames) { + if (!resource.attributes[propName] && !resource.properties[propName]) { + errors.push(`No such property in CFN Primary Identifiers Override file: ${resourceType}.${propName}`); + } + + resource.cfnRefIdentifier = propNames; + } + } catch (e: any) { + errors.push(e.message); + } + } + + if (errors.length > 0) { + throw new Error(errors.join('\n')); + } +} diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/index.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/index.ts index a95a20c460..594086080e 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loaders/index.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/index.ts @@ -8,3 +8,4 @@ export * from './load-sam-spec'; export * from './load-cloudwatch-console-service-directory'; export * from './load-getatt-allowlist'; export * from './load-oob-relationships'; +export * from './load-cfn-primary-identifier-overrides'; diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-cfn-primary-identifier-overrides.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cfn-primary-identifier-overrides.ts new file mode 100644 index 0000000000..ea1b54d6f8 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cfn-primary-identifier-overrides.ts @@ -0,0 +1,19 @@ +import { assertSuccess } from '@cdklabs/tskb'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; +import { CfnPrimaryIdentifierOverrides } from '../types'; + +export async function loadCfnPrimaryIdentifierOverrides( + filePath: string, + options: LoadSourceOptions = {}, +): Promise> { + const loader = await Loader.fromSchemaFile( + 'CfnPrimaryIdentifierOverrides.schema.json', + { + mustValidate: options.validate, + }, + ); + + const result = await loader.loadFile(filePath); + assertSuccess(result); + return result; +} diff --git a/packages/@aws-cdk/service-spec-importers/src/types/cfn-primaryidentifier-overrides/cfn-primaryidentifier-overrides.ts b/packages/@aws-cdk/service-spec-importers/src/types/cfn-primaryidentifier-overrides/cfn-primaryidentifier-overrides.ts new file mode 100644 index 0000000000..824661051d --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/types/cfn-primaryidentifier-overrides/cfn-primaryidentifier-overrides.ts @@ -0,0 +1,6 @@ +/** + * Maps a resource type to a list of properties that are also attributes + */ +export interface GetAttAllowList { + [resourceType: string]: string[]; +} diff --git a/packages/@aws-cdk/service-spec-importers/src/types/getatt-allowlist/getatt-allowlist.ts b/packages/@aws-cdk/service-spec-importers/src/types/getatt-allowlist/getatt-allowlist.ts index 824661051d..b7a5e539a2 100644 --- a/packages/@aws-cdk/service-spec-importers/src/types/getatt-allowlist/getatt-allowlist.ts +++ b/packages/@aws-cdk/service-spec-importers/src/types/getatt-allowlist/getatt-allowlist.ts @@ -1,6 +1,6 @@ /** - * Maps a resource type to a list of properties that are also attributes + * Maps a resource type to a list of properties that are overrides of the CCAPI schema primary identifiers */ -export interface GetAttAllowList { +export interface CfnPrimaryIdentifierOverrides { [resourceType: string]: string[]; } diff --git a/packages/@aws-cdk/service-spec-importers/src/types/index.ts b/packages/@aws-cdk/service-spec-importers/src/types/index.ts index 16a988753f..9153b60793 100644 --- a/packages/@aws-cdk/service-spec-importers/src/types/index.ts +++ b/packages/@aws-cdk/service-spec-importers/src/types/index.ts @@ -8,3 +8,4 @@ export * from './cloudwatch-console-service-directory/CloudWatchConsoleServiceDi export * from './getatt-allowlist/getatt-allowlist'; export * from './oob-relationships/OobRelationships'; export * from './eventbridge/EventBridgeSchema'; +export * from './cfn-primaryidentifier-overrides/cfn-primaryidentifier-overrides'; diff --git a/packages/@aws-cdk/service-spec-importers/test/import-cfn-identifiers.test.ts b/packages/@aws-cdk/service-spec-importers/test/import-cfn-identifiers.test.ts new file mode 100644 index 0000000000..0e2a7c4573 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/test/import-cfn-identifiers.test.ts @@ -0,0 +1,31 @@ +import { emptyDatabase, Resource } from '@aws-cdk/service-spec-types'; +import { importCfnPrimaryIdentifierOverrides } from '../src/importers/import-cfn-primaryidentifier-overrides'; + +let db: ReturnType; +beforeEach(() => { + db = emptyDatabase(); +}); + +test('exercise the CFN identifier import flow', () => { + db.allocate('resource', { + cloudFormationType: 'AWS::S3::Bucket', + attributes: {}, + name: 'Type', + primaryIdentifier: ['BucketName'], + properties: { + MyProp: { type: { type: 'string' } }, + }, + }); + + const overridesDocument = { + 'AWS::S3::Bucket': ['MyProp'], + }; + + importCfnPrimaryIdentifierOverrides(db, overridesDocument); + + const res = db.lookup('resource', 'cloudFormationType', 'equals', 'AWS::S3::Bucket').only(); + + expect(res).toMatchObject({ + cfnRefIdentifier: ['MyProp'], + } satisfies Partial); +}); diff --git a/packages/@aws-cdk/service-spec-types/src/types/diff.ts b/packages/@aws-cdk/service-spec-types/src/types/diff.ts index 4abf11c574..d61fd4e86f 100644 --- a/packages/@aws-cdk/service-spec-types/src/types/diff.ts +++ b/packages/@aws-cdk/service-spec-types/src/types/diff.ts @@ -45,7 +45,8 @@ export interface UpdatedResource { readonly vendedLogs?: ScalarDiff; readonly vendedLogsConfig?: ScalarDiff; readonly typeDefinitionDiff?: MapDiff; - readonly primaryIdentifier?: ListDiff; + readonly primaryIdentifier?: ScalarDiff; + readonly cfnRefIdentifier?: ScalarDiff; readonly metrics?: MapDiff; readonly events?: MapDiff; } diff --git a/packages/@aws-cdk/service-spec-types/src/types/resource.ts b/packages/@aws-cdk/service-spec-types/src/types/resource.ts index 405cd0ce2e..12cfb814ab 100644 --- a/packages/@aws-cdk/service-spec-types/src/types/resource.ts +++ b/packages/@aws-cdk/service-spec-types/src/types/resource.ts @@ -61,7 +61,28 @@ export interface Resource extends Entity { */ cloudFormationTransform?: string; documentation?: string; + + /** + * The primary identifier, or identifiers, in Cloud Control API + * + * Uniquely identifies a resource in an account. + * + * This is read from the schema, which always pertains to the CCAPI identifier. + * Typically the same as the CloudFormation identifier, but not necessarily. + */ primaryIdentifier?: string[]; + + /** + * The primary identifier, or identifiers, in CloudFormation + * + * Whatever gets returned when you call `{ Ref }` a resource. Typically the + * value that other resources in the same service expect to see as the way to + * reference this resource (name or ARN or Id). + * + * If missing, the `cfnRefIdentifier` is the same as the (CC-API) `primaryIdentifier`. + */ + cfnRefIdentifier?: string[]; + readonly properties: ResourceProperties; readonly attributes: Record; readonly validations?: unknown; @@ -303,11 +324,6 @@ export type ResourceInRegion = Relationship; export type UsesType = Relationship; -export interface ResourceIdentifier extends Entity { - readonly arnTemplate?: string; - readonly primaryIdentifier?: string[]; -} - /** * Mark a resource as a resource that needs additional scrutiy when added, removed or changed * diff --git a/sources/CloudFormationRefOverrides/README.md b/sources/CloudFormationRefOverrides/README.md new file mode 100644 index 0000000000..b79e535d58 --- /dev/null +++ b/sources/CloudFormationRefOverrides/README.md @@ -0,0 +1,13 @@ +# Summary + +This directory contains overrides for what the `{ Ref }` function returns in CloudFormation, if that disagrees +with the primary identifier from the schema. + +The schema describes the behavior of CloudControl API, which is the same as the +behavior of CloudFormation, except when it isn't. + +These are all validated by hand because there is no other automated source of truth for this information. + +# Motivation + +Since CDK is primarily built on top of CloudFormation, we need to know what CloudFormation actually does. \ No newline at end of file diff --git a/sources/CloudFormationRefOverrides/ref-overrides.json b/sources/CloudFormationRefOverrides/ref-overrides.json new file mode 100644 index 0000000000..76c1eac4b4 --- /dev/null +++ b/sources/CloudFormationRefOverrides/ref-overrides.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985331961594551d4272d07d4660d1e78d324e8eb6608faf35d11f1201dbad16 +size 3392