Rewrite dist => lib in serverless.ymls (#387)

skuba's default configuration compiles to `lib`, though it's common to
see `dist` and `lib` out in the wild. `skuba configure` already tries to
rewire Dockerfiles to point to this output directory, and now we do the
same for Serverless files.

This implements globbing in our configure modules so we can match on
multiple `serverless*.yml`s in the one repo.

Author: 72636c

.changeset/tall-crews-visit.md

@@ -0,0 +1,5 @@
+---
+'skuba': patch
+---
+
+**configure:** Rewrite `dist => lib` in `serverless.yml`s

package.json

@@ -45,6 +45,7 @@
     "@types/lodash.mergewith": "4.6.6",
     "@types/module-alias": "2.0.0",
     "@types/npm-which": "3.0.0",
+    "@types/picomatch": "2.2.1",
     "@types/supertest": "2.0.10",
     "express": "4.17.1",
     "koa": "2.13.1",
@@ -67,6 +68,7 @@
     "eslint": "^7.18.0",
     "eslint-config-skuba": "1.0.10",
     "execa": "^5.0.0",
+    "fdir": "^5.0.0",
     "fs-extra": "^9.1.0",
     "get-port": "^5.1.1",
     "ignore": "^5.1.8",
@@ -79,6 +81,7 @@
     "normalize-package-data": "^3.0.0",
     "npm-run-path": "^4.0.1",
     "npm-which": "^3.0.1",
+    "picomatch": "^2.2.2",
     "prettier": "2.2.1",
     "read-pkg-up": "^7.0.1",
     "runtypes": "^5.0.1",

src/cli/configure/analysis/project.test.ts

@@ -1,3 +1,4 @@
+import * as dir from '../../../utils/dir';
 import { getSkubaVersion } from '../../../utils/version';
 import { defaultOpts } from '../testing/module';
 
@@ -10,6 +11,8 @@ describe('diffFiles', () => {
       .spyOn(project, 'createDestinationFileReader')
       .mockReturnValue(() => Promise.resolve(undefined));
 
+    jest.spyOn(dir, 'crawlDirectory').mockResolvedValue([]);
+
     const [outputFiles, version] = await Promise.all([
       diffFiles(defaultOpts),
       getSkubaVersion(),

src/cli/configure/analysis/project.ts

@@ -2,6 +2,7 @@ import path from 'path';
 
 import fs from 'fs-extra';
 
+import { buildPatternToFilepathMap, crawlDirectory } from '../../../utils/dir';
 import { isErrorWithCode } from '../../../utils/error';
 import { loadModules } from '../modules';
 import { FileDiff, Files, Module, Options } from '../types';
@@ -25,30 +26,51 @@ export const createDestinationFileReader = (root: string) => async (
 const loadModuleFiles = async (modules: Module[], destinationRoot: string) => {
   const readDestinationFile = createDestinationFileReader(destinationRoot);
 
-  const allFilenames = modules.flatMap((module) => Object.keys(module));
+  const allFilepaths = await crawlDirectory(destinationRoot);
 
-  const uniqueFilenames = [...new Set(allFilenames)];
+  const patterns = [...new Set(modules.flatMap((m) => Object.keys(m)))];
+
+  const patternToFilepaths = buildPatternToFilepathMap(patterns, allFilepaths);
+
+  const matchedFilepaths = [
+    ...new Set(Object.values(patternToFilepaths).flat()),
+  ];
 
   const fileEntries = await Promise.all(
-    uniqueFilenames.map(
-      async (filename) =>
-        [filename, await readDestinationFile(filename)] as const,
+    matchedFilepaths.map(
+      async (filepath) =>
+        [filepath, await readDestinationFile(filepath)] as const,
     ),
   );
 
-  return Object.fromEntries(fileEntries);
+  return {
+    inputFiles: Object.fromEntries(fileEntries),
+    patternToFilepaths,
+  };
 };
 
-const processTextFiles = (modules: Module[], inputFiles: Readonly<Files>) => {
+const processTextFiles = (
+  modules: Module[],
+  inputFiles: Readonly<Files>,
+  patternToFilepaths: Record<string, string[]>,
+) => {
   const outputFiles = { ...inputFiles };
 
   const textProcessorEntries = modules.flatMap((module) =>
-    Object.entries(module),
+    Object.entries(module).flatMap(([pattern, processText]) => {
+      // Include the raw pattern along with any matched filepaths.
+      // Some modules create a new file at the specified pattern.
+      const filepaths = [pattern, ...(patternToFilepaths[pattern] ?? [])];
+
+      return [...new Set(filepaths)].map(
+        (filepath) => [filepath, processText] as const,
+      );
+    }),
   );
 
-  for (const [filename, processText] of textProcessorEntries) {
-    outputFiles[filename] = processText(
-      outputFiles[filename],
+  for (const [filepath, processText] of textProcessorEntries) {
+    outputFiles[filepath] = processText(
+      outputFiles[filepath],
       outputFiles,
       inputFiles,
     );
@@ -60,18 +82,18 @@ const processTextFiles = (modules: Module[], inputFiles: Readonly<Files>) => {
 export const diffFiles = async (opts: Options): Promise<FileDiff> => {
   const modules = await loadModules(opts);
 
-  const inputFiles = Object.freeze(
+  const { inputFiles, patternToFilepaths } = Object.freeze(
     await loadModuleFiles(modules, opts.destinationRoot),
   );
 
-  const outputFiles = processTextFiles(modules, inputFiles);
+  const outputFiles = processTextFiles(modules, inputFiles, patternToFilepaths);
 
   const diffEntries = Object.entries(outputFiles)
-    .filter(([filename, data]) => inputFiles[filename] !== data)
-    .map(([filename, data]) => {
-      const operation = determineOperation(inputFiles[filename], data);
+    .filter(([filepath, data]) => inputFiles[filepath] !== data)
+    .map(([filepath, data]) => {
+      const operation = determineOperation(inputFiles[filepath], data);
 
-      return [filename, { data, operation }] as const;
+      return [filepath, { data, operation }] as const;
     });
 
   return Object.fromEntries(diffEntries);

src/cli/configure/modules/index.ts

@@ -7,6 +7,7 @@ import { nodemonModule } from './nodemon';
 import { packageModule } from './package';
 import { prettierModule } from './prettier';
 import { renovateModule } from './renovate';
+import { serverlessModule } from './serverless';
 import { skubaDiveModule } from './skubaDive';
 import { tsconfigModule } from './tsconfig';
 import { tslintModule } from './tslint';
@@ -21,6 +22,7 @@ export const loadModules = (opts: Options): Promise<Module[]> =>
       packageModule,
       prettierModule,
       renovateModule,
+      serverlessModule,
       skubaDiveModule,
       tsconfigModule,
       tslintModule,

src/cli/configure/modules/serverless.test.ts

@@ -0,0 +1,123 @@
+import { defaultOpts, executeModule } from '../testing/module';
+
+import { serverlessModule } from './serverless';
+
+describe('serverlessModule', () => {
+  it('does not touch empty input', async () => {
+    const inputFiles = {};
+
+    const outputFiles = await executeModule(
+      serverlessModule,
+      inputFiles,
+      defaultOpts,
+    );
+
+    expect(outputFiles).toEqual({});
+  });
+
+  it('rewires a variety of patterns', async () => {
+    const INPUT_SERVERLESS_YAML = `
+      service: unrelated-dist-reference
+
+      package:
+        include:
+          - dist/**
+
+      package:
+        include:
+          - /dist/**
+
+      package:
+        include:
+          - ./dist/**
+
+      functions:
+        WorkerA:
+          handler: dist/app.aHandler
+        WorkerB:
+          handler: /dist/app.bHandler
+        WorkerC:
+          handler: ./dist/app.cHandler
+    `;
+
+    const EXPECTED_SERVERLESS_YAML = `
+      service: unrelated-dist-reference
+
+      package:
+        include:
+          - lib/**
+
+      package:
+        include:
+          - lib/**
+
+      package:
+        include:
+          - lib/**
+
+      functions:
+        WorkerA:
+          handler: lib/app.aHandler
+        WorkerB:
+          handler: lib/app.bHandler
+        WorkerC:
+          handler: lib/app.cHandler
+    `;
+
+    const inputFiles = {
+      'serverless.yml': INPUT_SERVERLESS_YAML,
+    };
+
+    const outputFiles = await executeModule(
+      serverlessModule,
+      inputFiles,
+      defaultOpts,
+    );
+
+    expect(outputFiles).toEqual({
+      'serverless.yml': EXPECTED_SERVERLESS_YAML,
+    });
+  });
+
+  it('rewires a variety of Serverless files', async () => {
+    const INPUT_SERVERLESS_YAML = `
+      package:
+        include:
+          - dist/**
+
+      functions:
+        Worker:
+          handler: dist/app.handler
+    `;
+
+    const EXPECTED_SERVERLESS_YAML = `
+      package:
+        include:
+          - lib/**
+
+      functions:
+        Worker:
+          handler: lib/app.handler
+    `;
+
+    const inputFiles = {
+      'serverless.yml': INPUT_SERVERLESS_YAML,
+      'serverless.service.yml': INPUT_SERVERLESS_YAML,
+      'service/serverless.yml': INPUT_SERVERLESS_YAML,
+      'unrelated.txt': 'dist',
+    };
+
+    const outputFiles = await executeModule(
+      serverlessModule,
+      inputFiles,
+      defaultOpts,
+    );
+
+    expect(outputFiles).toEqual({
+      'serverless.yml': EXPECTED_SERVERLESS_YAML,
+      'serverless.service.yml': EXPECTED_SERVERLESS_YAML,
+      'service/serverless.yml': EXPECTED_SERVERLESS_YAML,
+      'unrelated.txt': 'dist',
+    });
+  });
+});

src/cli/configure/modules/serverless.ts

@@ -0,0 +1,18 @@
+import { Module, Options } from '../types';
+
+export const serverlessModule = ({}: Options): Module => ({
+  '**/serverless*.yml': (inputFile, _files, _initialFiles) => {
+    if (!inputFile) {
+      // Only configure files that exist.
+      return;
+    }
+
+    return (
+      inputFile
+        // Rewire packaging patterns.
+        .replace(/- (\.?\/)?dist\//g, '- lib/')
+        // Rewire handler paths.
+        .replace(/handler: (\.?\/)?dist\//g, 'handler: lib/')
+    );
+  },
+});

src/cli/configure/testing/module.ts

@@ -1,3 +1,5 @@
+import picomatch from 'picomatch';
+
 import { Files, Module, Options } from '../types';
 
 export function assertDefined<T>(value?: T): asserts value is T {
@@ -27,12 +29,22 @@ export const executeModule = async (
 
   const outputFiles = { ...inputFiles };
 
-  for (const [filename, processText] of Object.entries(mod)) {
-    outputFiles[filename] = processText(
-      outputFiles[filename],
-      outputFiles,
-      inputFiles,
-    );
+  const allFilepaths = Object.keys(outputFiles);
+
+  for (const [pattern, processText] of Object.entries(mod)) {
+    const isMatch = picomatch(pattern);
+
+    // Include the raw pattern along with any matched filepaths.
+    // Some modules create a new file at the specified pattern.
+    const filepaths = [pattern, ...allFilepaths.filter((p) => isMatch(p))];
+
+    for (const filepath of [...new Set(filepaths)]) {
+      outputFiles[filepath] = processText(
+        outputFiles[filepath],
+        outputFiles,
+        inputFiles,
+      );
+    }
   }
 
   return outputFiles;