chore(spec2cdk): refactor to make it easier to extend

The `spec2cdk` tool was originally written to generate every service
submodule insularly.

We want to now extend it to emit structures into a new shared submodule.
In order to do that, this refactors the generator to have a more shared,
mutable object-oriented design which will be easier to share state in.

In addition, simplify the API around file name patterns, because they
were being overly flexible and annoying to work with for no real
benefit.

Author: rix0rrr

tools/@aws-cdk/spec2cdk/lib/cdk/ast.ts

@@ -5,32 +5,9 @@ import { CannedMetricsModule } from './canned-metrics';
 import { CDK_CORE, CONSTRUCTS, ModuleImportLocations } from './cdk';
 import { SelectiveImport } from './relationship-decider';
 import { ResourceClass } from './resource-class';
+import { FilePatternValues, TsFileWriter, substituteFilePattern } from '../util';
 
-/**
- * A module containing a single resource
- */
-export class ResourceModule extends Module {
-  public constructor(public readonly service: string, public readonly resource: string) {
-    super(`@aws-cdk/${service}/${resource}-l1`);
-  }
-}
-
-/**
- * A module containing a service
- */
-export class ServiceModule extends Module {
-  public constructor(public readonly service: string, public readonly shortName: string) {
-    super(`@aws-cdk/${service}`);
-  }
-}
-
-export interface AstBuilderProps {
-  readonly db: SpecDatabase;
-  /**
-   * Override the locations modules are imported from
-   */
-  readonly importLocations?: ModuleImportLocations;
-
+export interface AddServiceProps {
   /**
    * Append a suffix at the end of generated names.
    */
@@ -42,82 +19,126 @@ export interface AstBuilderProps {
    * @default - not deprecated
    */
   readonly deprecated?: string;
-}
 
-export class AstBuilder<T extends Module> {
   /**
-   * Build a module for all resources in a service
+   * The target resource module we want to generate these resources into
+   *
+   * (Practically, only used to render CloudFormation resources into the `core` module.)
    */
-  public static forService(service: Service, props: AstBuilderProps): AstBuilder<ServiceModule> {
-    const scope = new ServiceModule(service.name, service.shortName);
-    const aug = new AugmentationsModule(props.db, service.name, props.importLocations?.cloudwatch);
-    const metrics = CannedMetricsModule.forService(props.db, service);
+  readonly destinationModule?: string;
 
-    const ast = new AstBuilder(scope, props, aug, metrics);
+  /**
+   * Override the locations modules are imported from
+   */
+  readonly importLocations?: ModuleImportLocations;
+}
 
-    const resources = props.db.follow('hasResource', service);
+export interface AstBuilderProps {
+  readonly db: SpecDatabase;
 
-    for (const link of resources) {
-      ast.addResource(link.entity);
-    }
-    ast.renderImports();
-    return ast;
-  }
+  readonly modulesRoot?: string;
+}
 
+export interface GenerateFilePatterns {
   /**
-   * Build an module for a single resource
+   * The pattern used to name resource files.
+   * @default "%serviceName%/%serviceShortName%.generated.ts"
    */
-  public static forResource(resource: Resource, props: AstBuilderProps): AstBuilder<ResourceModule> {
-    const parts = resource.cloudFormationType.toLowerCase().split('::');
-    const scope = new ResourceModule(parts[1], parts[2]);
-    const aug = new AugmentationsModule(props.db, parts[1], props.importLocations?.cloudwatch);
-    const metrics = CannedMetricsModule.forResource(props.db, resource);
-
-    const ast = new AstBuilder(scope, props, aug, metrics);
-    ast.addResource(resource);
-    ast.renderImports();
+  readonly resources: string;
 
-    return ast;
-  }
+  /**
+   * The pattern used to name augmentations.
+   * @default "%serviceName%/%serviceShortName%-augmentations.generated.ts"
+   */
+  readonly augmentations: string;
 
-  public readonly db: SpecDatabase;
   /**
-   * Map of CloudFormation resource name to generated class name
+   * The pattern used to name canned metrics.
+   * @default "%serviceName%/%serviceShortName%-canned-metrics.generated.ts"
    */
-  public readonly resources: Record<string, string> = {};
-  private nameSuffix?: string;
-  private deprecated?: string;
+  readonly cannedMetrics: string;
+}
+
+export const DEFAULT_FILE_PATTERNS: GenerateFilePatterns = {
+  resources: '%serviceName%/%serviceShortName%.generated.ts',
+  augmentations: '%serviceName%/%serviceShortName%-augmentations.generated.ts',
+  cannedMetrics: '%serviceName%/%serviceShortName%-canned-metrics.generated.ts',
+};
+
+export class AstBuilder {
+  public readonly db: SpecDatabase;
+
   public readonly selectiveImports = new Array<SelectiveImport>();
+
+  public readonly serviceModules = new Map<string, SubmoduleInfo>();
   private readonly modulesRootLocation: string;
 
-  protected constructor(
-    public readonly module: T,
+  constructor(
     props: AstBuilderProps,
-    public readonly augmentations?: AugmentationsModule,
-    public readonly cannedMetrics?: CannedMetricsModule,
   ) {
     this.db = props.db;
-    this.nameSuffix = props.nameSuffix;
-    this.deprecated = props.deprecated;
-    this.modulesRootLocation = props.importLocations?.modulesRoot ?? '../..';
-
-    CDK_CORE.import(this.module, 'cdk', { fromLocation: props.importLocations?.core });
-    CONSTRUCTS.import(this.module, 'constructs');
-    CDK_CORE.helpers.import(this.module, 'cfn_parse', { fromLocation: props.importLocations?.coreHelpers });
-    CDK_CORE.errors.import(this.module, 'cdk_errors', { fromLocation: props.importLocations?.coreErrors });
+    this.modulesRootLocation = props.modulesRoot ?? '../..';
   }
 
-  public addResource(resource: Resource) {
-    const resourceClass = new ResourceClass(this.module, this.db, resource, {
-      suffix: this.nameSuffix,
-      deprecated: this.deprecated,
+  /**
+   * Add all resources in a service
+   */
+  public addService(service: Service, props?: AddServiceProps) {
+    const resources = this.db.follow('hasResource', service);
+    const submod = this.createSubmodule(service, props?.destinationModule, props?.importLocations);
+
+    for (const { entity: resource } of resources) {
+      this.addResourceToSubmodule(submod, resource, props);
+    }
+
+    this.renderImports(submod);
+    return submod;
+  }
+
+  /**
+   * Build an module for a single resource (only used for testing)
+   */
+  public addResource(resource: Resource, props?: AddServiceProps) {
+    const service = this.db.incoming('hasResource', resource).only().entity;
+    const submod = this.createSubmodule(service, props?.destinationModule, props?.importLocations);
+
+    this.addResourceToSubmodule(submod, resource, props);
+
+    this.renderImports(submod);
+    return submod;
+  }
+
+  public writeAll(writer: TsFileWriter, filePatterns: GenerateFilePatterns) {
+    for (const mods of this.serviceModules.values()) {
+      const pattern: FilePatternValues = {
+        serviceName: mods.service.name,
+        serviceShortName: mods.service.shortName,
+        moduleName: mods.moduleName,
+      };
+
+      writer.write(mods.resourceModule, substituteFilePattern(filePatterns.resources, pattern));
+
+      if (mods.augmentationsModule.hasAugmentations) {
+        writer.write(mods.augmentationsModule, substituteFilePattern(filePatterns.augmentations, pattern));
+      }
+
+      if (mods.cannedMetricsModule.hasCannedMetrics) {
+        writer.write(mods.cannedMetricsModule, substituteFilePattern(filePatterns.cannedMetrics, pattern));
+      }
+    }
+  }
+
+  private addResourceToSubmodule(submodule: SubmoduleInfo, resource: Resource, props?: AddServiceProps) {
+    const resourceClass = new ResourceClass(submodule.resourceModule, this.db, resource, {
+      suffix: props?.nameSuffix,
+      deprecated: props?.deprecated,
     });
-    this.resources[resource.cloudFormationType] = resourceClass.spec.name;
+    submodule.resources[resource.cloudFormationType] = resourceClass.spec.name;
 
     resourceClass.build();
 
     this.addImports(resourceClass);
-    this.augmentations?.augmentResource(resource, resourceClass);
+    submodule.augmentationsModule.augmentResource(resource, resourceClass);
   }
 
   private addImports(resourceClass: ResourceClass) {
@@ -140,11 +161,61 @@ export class AstBuilder<T extends Module> {
     }
   }
 
-  public renderImports() {
+  private renderImports(serviceModules: SubmoduleInfo) {
     const sortedImports = this.selectiveImports.sort((a, b) => a.moduleName.localeCompare(b.moduleName));
     for (const selectiveImport of sortedImports) {
       const sourceModule = new Module(selectiveImport.moduleName);
-      sourceModule.importSelective(this.module, selectiveImport.types.map((t) => `${t.originalType} as ${t.aliasedType}`), { fromLocation: `${this.modulesRootLocation}/${sourceModule.name}` });
+      sourceModule.importSelective(serviceModules.resourceModule, selectiveImport.types.map((t) => `${t.originalType} as ${t.aliasedType}`), {
+        fromLocation: `${this.modulesRootLocation}/${sourceModule.name}`,
+      });
     }
   }
+
+  private createSubmodule(service: Service, targetServiceModule?: string, importLocations?: ModuleImportLocations): SubmoduleInfo {
+    const moduleName = targetServiceModule ?? service.name;
+
+    const mods = this.serviceModules.get(moduleName);
+    if (mods) {
+      // eslint-disable-next-line @cdklabs/no-throw-default-error
+      throw new Error(`A submodule named ${moduleName} was already created`);
+    }
+
+    const resourceModule = new Module(`@aws-cdk/${moduleName}`);
+    CDK_CORE.import(resourceModule, 'cdk', { fromLocation: importLocations?.core });
+    CONSTRUCTS.import(resourceModule, 'constructs');
+    CDK_CORE.helpers.import(resourceModule, 'cfn_parse', { fromLocation: importLocations?.coreHelpers });
+    CDK_CORE.errors.import(resourceModule, 'cdk_errors', { fromLocation: importLocations?.coreErrors });
+
+    const augmentationsModule = new AugmentationsModule(this.db, service.shortName, importLocations?.cloudwatch);
+    const cannedMetricsModule = CannedMetricsModule.forService(this.db, service);
+
+    const ret: SubmoduleInfo = {
+      service,
+      moduleName,
+      resourceModule,
+      augmentationsModule,
+      cannedMetricsModule,
+      resources: {},
+    };
+    this.serviceModules.set(moduleName, ret);
+    return ret;
+  }
+}
+
+export interface SubmoduleInfo {
+  readonly service: Service;
+
+  /**
+   * The name of the submodule of aws-cdk-lib
+   */
+  readonly moduleName: string;
+
+  readonly resourceModule: Module;
+  readonly augmentationsModule: AugmentationsModule;
+  readonly cannedMetricsModule: CannedMetricsModule;
+
+  /**
+   * Map of CloudFormation resource name to generated class name
+   */
+  readonly resources: Record<string, string>;
 }

tools/@aws-cdk/spec2cdk/lib/cfn2ts/index.ts

@@ -10,11 +10,6 @@ import { log } from '../util';
 
 export * from './types';
 
-interface GenerateOutput {
-  outputFiles: string[];
-  resources: Record<string, string>;
-}
-
 let serviceCache: Service[];
 
 async function getAllScopes(field: keyof Service = 'name'): Promise<ModuleMapScope[]> {
@@ -30,7 +25,7 @@ export default async function generate(
   scopes: string | string[],
   outPath: string,
   options: CodeGeneratorOptions = {},
-): Promise<GenerateOutput> {
+): Promise<void> {
   const coreImport = options.coreImport ?? 'aws-cdk-lib';
   let moduleScopes: ModuleMapScope[] = [];
   if (scopes === '*') {
@@ -40,7 +35,7 @@ export default async function generate(
   }
 
   log.info(`cfn-resources: ${moduleScopes.map(s => s.namespace).join(', ')}`);
-  const generated = await generateModules(
+  await generateModules(
     {
       'aws-cdk-lib': {
         services: options.autoGenerateSuffixes ? computeServiceSuffixes(moduleScopes) : moduleScopes,
@@ -50,9 +45,9 @@ export default async function generate(
       outputPath: outPath ?? 'lib',
       clearOutput: false,
       filePatterns: {
-        resources: ({ serviceShortName }) => `${serviceShortName}.generated.ts`,
-        augmentations: ({ serviceShortName }) => `${serviceShortName}-augmentations.generated.ts`,
-        cannedMetrics: ({ serviceShortName }) => `${serviceShortName}-canned-metrics.generated.ts`,
+        resources: '%serviceShortName.generated.ts',
+        augmentations: '%serviceShortName%-augmentations.generated.ts',
+        cannedMetrics: '%serviceShortName%-canned-metrics.generated.ts',
       },
       importLocations: {
         core: coreImport,
@@ -61,8 +56,6 @@ export default async function generate(
       },
     },
   );
-
-  return generated;
 }
 
 /**
@@ -163,9 +156,9 @@ export async function generateAll(
       outputPath: outPath,
       clearOutput: false,
       filePatterns: {
-        resources: ({ moduleName: m, serviceShortName: s }) => `${m}/lib/${s}.generated.ts`,
-        augmentations: ({ moduleName: m, serviceShortName: s }) => `${m}/lib/${s}-augmentations.generated.ts`,
-        cannedMetrics: ({ moduleName: m, serviceShortName: s }) => `${m}/lib/${s}-canned-metrics.generated.ts`,
+        resources: '%moduleName%/lib/%serviceShortName%.generated.ts',
+        augmentations: '%moduleName%/lib/%serviceShortName%-augmentations.generated.ts',
+        cannedMetrics: '%moduleName%/lib/%serviceShortName%-canned-metrics.generated.ts',
       },
       importLocations: {
         core: options.coreImport,