hld lifecycle pipeline to blow away repository directory in hld before regenerating assets (#505)

* Adding purgeRepositoryComponent to reconcile step to remove existing files on hld with some exceptions

* util for getting all files in directory and subdirectories

* Adding purging of project directories for reconcile in hld repository

* Tested logic locally, looks to be working. Need to add unit tests and possibly ITs

* tests

* feedback

Co-authored-by: Bhargav Nookala <[email protected]>

Author: mtarng

src/commands/hld/reconcile.test.ts

@@ -16,11 +16,14 @@ import {
   execute,
   getFullPathPrefix,
   normalizedName,
+  purgeRepositoryComponents,
   ReconcileDependencies,
   reconcileHld,
   testAndGetAbsPath,
   validateInputs,
 } from "./reconcile";
+import mockFs from "mock-fs";
+import fs from "fs";
 
 beforeAll(() => {
   enableVerboseLogging();
@@ -169,6 +172,109 @@ describe("createAccessYaml", () => {
   });
 });
 
+describe("purgeRepositoryComponents", () => {
+  const fsSpy = jest.spyOn(fs, "unlink");
+
+  beforeEach(() => {
+    mockFs({
+      "hld-repo": {
+        config: {
+          "common.yaml": "someconfigfile",
+        },
+        "bedrock-project-repo": {
+          "access.yaml": "someaccessfile",
+          config: {
+            "common.yaml": "someconfigfile",
+          },
+          serviceA: {
+            config: {
+              "common.yaml": "someconfigfile",
+            },
+            master: {
+              config: {
+                "common.yaml": "someconfigfile",
+                "prod.yaml": "someconfigfile",
+                "stage.yaml": "someconfigfile",
+              },
+              static: {
+                "ingressroute.yaml": "ingressroutefile",
+                "middlewares.yaml": "middlewaresfile",
+              },
+              "component.yaml": "somecomponentfile",
+            },
+            "component.yaml": "somecomponentfile",
+          },
+          "component.yaml": "somecomponentfile",
+        },
+        "component.yaml": "somecomponentfile",
+      },
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+    jest.clearAllMocks();
+    fsSpy.mockClear();
+  });
+
+  const hldPath = "hld-repo";
+  const repositoryName = "bedrock-project-repo";
+
+  it("should invoke fs.unlink for each file in project repository except config files and access.yaml", async () => {
+    purgeRepositoryComponents(hldPath, repositoryName);
+
+    expect(fs.unlink).toHaveBeenCalledTimes(5);
+    expect(fs.unlink).toHaveBeenCalledWith(
+      path.join(hldPath, repositoryName, "component.yaml"),
+      expect.any(Function)
+    );
+    expect(fs.unlink).toHaveBeenCalledWith(
+      path.join(hldPath, repositoryName, "serviceA", "component.yaml"),
+      expect.any(Function)
+    );
+    expect(fs.unlink).toHaveBeenCalledWith(
+      path.join(
+        hldPath,
+        repositoryName,
+        "serviceA",
+        "master",
+        "component.yaml"
+      ),
+      expect.any(Function)
+    );
+    expect(fs.unlink).toHaveBeenCalledWith(
+      path.join(
+        hldPath,
+        repositoryName,
+        "serviceA",
+        "master",
+        "static",
+        "middlewares.yaml"
+      ),
+      expect.any(Function)
+    );
+    expect(fs.unlink).toHaveBeenCalledWith(
+      path.join(
+        hldPath,
+        repositoryName,
+        "serviceA",
+        "master",
+        "static",
+        "ingressroute.yaml"
+      ),
+      expect.any(Function)
+    );
+  });
+
+  it("should throw an error if fs fails", async () => {
+    fsSpy.mockImplementationOnce(() => {
+      throw Error("some error");
+    });
+
+    expect(() => purgeRepositoryComponents(hldPath, repositoryName)).toThrow();
+  });
+});
+
 describe("createRepositoryComponent", () => {
   let exec = jest.fn().mockReturnValue(Promise.resolve({}));
   const hldPath = `myMonoRepo`;
@@ -446,6 +552,7 @@ describe("reconcile tests", () => {
       exec: jest.fn().mockReturnValue(Promise.resolve({})),
       generateAccessYaml: jest.fn(),
       getGitOrigin: jest.fn(),
+      purgeRepositoryComponents: jest.fn(),
       writeFile: jest.fn(),
     };
 
@@ -494,6 +601,7 @@ describe("reconcile tests", () => {
     expect(dependencies.createMiddlewareForRing).toHaveBeenCalledTimes(2);
     expect(dependencies.createIngressRouteForRing).toHaveBeenCalledTimes(2);
     expect(dependencies.generateAccessYaml).toHaveBeenCalledTimes(1);
+    expect(dependencies.purgeRepositoryComponents).toHaveBeenCalledTimes(1);
     expect(dependencies.generateAccessYaml).toBeCalledWith(
       "path/to/hld/service",
       git,

src/commands/hld/reconcile.ts

@@ -19,6 +19,7 @@ import { BedrockFile, BedrockServiceConfig } from "../../types";
 import decorator from "./reconcile.decorator.json";
 import { build as buildError, log as logError } from "../../lib/errorBuilder";
 import { errorStatusCode } from "../../lib/errorStatusCode";
+import { getAllFilesInDirectory } from "../../lib/ioUtil";
 
 /**
  * IExecResult represents the possible return value of a Promise based wrapper
@@ -101,6 +102,46 @@ type MiddlewareMap<T = Partial<ReturnType<typeof middleware.create>>> = {
   default?: T;
 };
 
+/**
+ * In spk hld reconcile, the results should always result in the same artifacts being created based on the state of bedrock.yaml.
+ * The only exception is for files under the /config directories and any access.yaml files.
+ * @param absHldPath Absolute path to the local HLD repository directory
+ * @param repositoryName Name of the bedrock project repository/directory inside of the HLD repository
+ */
+export const purgeRepositoryComponents = (
+  absHldPath: string,
+  repositoryName: string
+): void => {
+  assertIsStringWithContent(absHldPath, "hld-path");
+  assertIsStringWithContent(repositoryName, "repository-name");
+
+  const filesToDelete = getAllFilesInDirectory(
+    path.join(absHldPath, repositoryName)
+  ).filter(
+    (filePath) =>
+      !filePath.endsWith("access.yaml") && !filePath.match(/config\/.*\.yaml$/)
+  );
+
+  try {
+    filesToDelete.forEach((file) => {
+      fs.unlink(file, function (err) {
+        if (err) throw err;
+        console.log(`${file} deleted!`);
+      });
+    });
+  } catch (err) {
+    throw buildError(
+      errorStatusCode.FILE_IO_ERR,
+      {
+        errorKey: "hld-reconcile-err-purge-repo-comps",
+        values: [repositoryName, absHldPath],
+      },
+      err
+    );
+  }
+  return;
+};
+
 export const createRepositoryComponent = async (
   execCmd: typeof execAndLog,
   absHldPath: string,
@@ -381,6 +422,7 @@ export interface ReconcileDependencies {
   getGitOrigin: typeof tryGetGitOrigin;
   generateAccessYaml: typeof generateAccessYaml;
   createAccessYaml: typeof createAccessYaml;
+  purgeRepositoryComponents: typeof purgeRepositoryComponents;
   createRepositoryComponent: typeof createRepositoryComponent;
   configureChartForRing: typeof configureChartForRing;
   createServiceComponent: typeof createServiceComponent;
@@ -454,6 +496,12 @@ export const reconcileHld = async (
 ): Promise<void> => {
   const { services: managedServices, rings: managedRings } = bedrockYaml;
 
+  // To support removing services and rings, first remove all files under an application repository directory except anything in a /config directory and any access.yaml files, then we generate all values again.
+  dependencies.purgeRepositoryComponents(
+    absHldPath,
+    normalizedName(repositoryName)
+  );
+
   // Create Repository Component if it doesn't exist.
   // In a pipeline, the repository component is the name of the application repository.
   await dependencies.createRepositoryComponent(
@@ -649,6 +697,7 @@ export const execute = async (
       exec: execAndLog,
       generateAccessYaml,
       getGitOrigin: tryGetGitOrigin,
+      purgeRepositoryComponents,
       writeFile: fs.writeFileSync,
     };
 

src/lib/i18n.json

@@ -31,6 +31,7 @@
     "hld-reconcile-err-path": "Could not validate {0} path.",
     "hld-reconcile-err-cmd-failed": "An error occurred executing command: {0}",
     "hld-reconcile-err-cmd-exe": "An error occurred while reconciling HLD.",
+    "hld-reconcile-err-purge-repo-comps": "Could not purge hld repository component {0} in path {1}.",
 
     "infra-scaffold-cmd-failed": "Infra scaffold command was not successfully executed.",
     "infra-scaffold-cmd-src-missing": "Value for source is required because it cannot be constructed with properties in spk-config.yaml. Provide value for source.",

src/lib/ioUtil.test.ts

@@ -3,10 +3,25 @@ import path from "path";
 import uuid from "uuid/v4";
 import {
   createTempDir,
+  getAllFilesInDirectory,
   getMissingFilenames,
   isDirEmpty,
   removeDir,
 } from "./ioUtil";
+import mockFs from "mock-fs";
+import { disableVerboseLogging, enableVerboseLogging, logger } from "../logger";
+
+beforeAll(() => {
+  enableVerboseLogging();
+});
+
+afterAll(() => {
+  disableVerboseLogging();
+});
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
 
 describe("test createTempDir function", () => {
   it("create and existence check", () => {
@@ -110,3 +125,69 @@ describe("test doFilesExist function", () => {
     expect(missing.length).toBe(1);
   });
 });
+
+describe("test getAllFilesInDirectory function", () => {
+  beforeEach(() => {
+    mockFs({
+      "hld-repo": {
+        config: {
+          "common.yaml": "someconfigfile",
+        },
+        "bedrock-project-repo": {
+          "access.yaml": "someaccessfile",
+          config: {
+            "common.yaml": "someconfigfile",
+          },
+          serviceA: {
+            config: {
+              "common.yaml": "someconfigfile",
+            },
+            master: {
+              config: {
+                "common.yaml": "someconfigfile",
+                "prod.yaml": "someconfigfile",
+                "stage.yaml": "someconfigfile",
+              },
+              static: {
+                "ingressroute.yaml": "ingressroutefile",
+                "middlewares.yaml": "middlewaresfile",
+              },
+              "component.yaml": "somecomponentfile",
+            },
+            "component.yaml": "somecomponentfile",
+          },
+          "component.yaml": "somecomponentfile",
+        },
+        "component.yaml": "somecomponentfile",
+      },
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it("gets all files in a populated directory", () => {
+    const fileList = getAllFilesInDirectory("hld-repo");
+    expect(fileList).toHaveLength(13);
+    expect(fileList).toContain(
+      "hld-repo/bedrock-project-repo/serviceA/master/static/middlewares.yaml"
+    );
+
+    const filesToDelete = fileList.filter(
+      (filePath) =>
+        !filePath.match(/access\.yaml$/) && !filePath.match(/config\/.*\.yaml$/)
+    );
+    logger.info("filestoDelete.length: " + filesToDelete.length);
+    logger.info(filesToDelete);
+  });
+
+  it("returns an empty list when there's no files directory", () => {
+    mockFs({
+      emptyDirectory: {},
+    });
+
+    const fileList = getAllFilesInDirectory("emptyDirectory");
+    expect(fileList).toHaveLength(0);
+  });
+});

src/lib/ioUtil.ts

@@ -2,6 +2,7 @@ import fs from "fs";
 import os from "os";
 import path from "path";
 import uuid from "uuid/v4";
+import { logger } from "../logger";
 
 /**
  * Creates a random directory in tmp directory.
@@ -68,3 +69,27 @@ export const getMissingFilenames = (
     )
     .map((f) => path.basename(f));
 };
+
+/**
+ * Returns all files/filepaths in this directory and subdirectories.
+ *
+ * @param dir directory to search
+ * @param files list of files
+ * @returns list of files in given directory and subdirectories.
+ */
+export const getAllFilesInDirectory = (
+  dir: string,
+  files: string[] = []
+): string[] => {
+  const filesInDir = fs.readdirSync(dir);
+
+  filesInDir.forEach(function (file) {
+    if (fs.statSync(path.join(dir, file)).isDirectory()) {
+      files = getAllFilesInDirectory(path.join(dir, file), files);
+    } else {
+      files.push(path.join(dir, file));
+    }
+  });
+
+  return files;
+};