fix: Move persistence layer's delta computation into worker

Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { FileAsset } from '../persistence/types';
+
+import Worker from './workers/calculator.worker.ts';
+import { BaseWorker } from './baseWorker';
+import { FilesDifferencePayload, CalculatorType } from './types';
+
+// Wrapper class
+class FileDiffCalculator extends BaseWorker<CalculatorType> {
+  difference(target: FileAsset[], origin: FileAsset[]) {
+    const payload = { target, origin };
+    return this.sendMsg<FilesDifferencePayload>('difference', payload);
+  }
+}
+
+export default new FileDiffCalculator(new Worker());

Composer/packages/client/src/recoilModel/parsers/types.ts

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 import { LuIntentSection, LgFile, LuFile, QnASection, FileInfo, LgTemplate, ILUFeaturesConfig } from '@bfc/shared';
 
+import { FileAsset } from '../persistence/types';
+
 export type LuParsePayload = {
   id: string;
   content: string;
@@ -144,3 +146,10 @@ export enum QnAActionType {
   UpdateSection = 'update-section',
   RemoveSection = 'remove-section',
 }
+
+export type FilesDifferencePayload = {
+  target: FileAsset[];
+  origin: FileAsset[];
+};
+
+export type CalculatorType = 'difference';

Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import isEqual from 'lodash/isEqual';
+import differenceWith from 'lodash/differenceWith';
+
+import { FileAsset } from '../../persistence/types';
+
+import { CalculatorType, FilesDifferencePayload } from './../types';
+
+type DifferenceMessage = {
+  id: string;
+  type: CalculatorType;
+  payload: FilesDifferencePayload;
+};
+
+type MessageEvent = DifferenceMessage;
+
+const ctx: Worker = self as any;
+
+export function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) {
+  const changes1 = differenceWith(target, origin, isEqual);
+  const changes2 = differenceWith(origin, target, isEqual);
+  const deleted = changes2.filter((change) => !target.some((file) => change.id === file.id));
+  const { updated, added } = changes1.reduce(
+    (result: { updated: FileAsset[]; added: FileAsset[] }, change) => {
+      if (origin.some((file) => change.id === file.id)) {
+        result.updated.push(change);
+      } else {
+        result.added.push(change);
+      }
+
+      return result;
+    },
+    { updated: [], added: [] }
+  );
+
+  return { updated, added, deleted };
+}
+
+function handleMessage(message: MessageEvent) {
+  const { type, payload } = message;
+
+  switch (type) {
+    case 'difference': {
+      const { target, origin } = payload;
+      return getDifferenceItems(target, origin);
+    }
+    default: {
+      return null;
+    }
+  }
+}
+
+ctx.onmessage = function (event) {
+  const msg = event.data as MessageEvent;
+  try {
+    const result = handleMessage(msg);
+    ctx.postMessage({ id: msg.id, payload: result });
+  } catch (error) {
+    ctx.postMessage({ id: msg.id, error: error.message });
+  }
+};

Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts

@@ -1,26 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-
-import differenceWith from 'lodash/differenceWith';
 import isEqual from 'lodash/isEqual';
-import {
-  DialogInfo,
-  DialogSchemaFile,
-  DialogSetting,
-  BotAssets,
-  BotProjectFile,
-  LuFile,
-  LgFile,
-  QnAFile,
-  FormDialogSchema,
-  RecognizerFile,
-  CrosstrainConfig,
-  SkillManifestFile,
-} from '@bfc/shared';
+import { DialogSetting, BotAssets, BotProjectFile, CrosstrainConfig } from '@bfc/shared';
 import keys from 'lodash/keys';
 
+import fileDiffCalculator from '../parsers/fileDiffCalculator';
+
 import * as client from './http';
-import { ChangeType, FileExtensions, IFileChange } from './types';
+import { ChangeType, FileDifference, FileExtensions, IFileChange, FileAsset } from './types';
 
 class FilePersistence {
   private _taskQueue: { [id: string]: IFileChange[] } = {};
@@ -47,16 +34,20 @@ class FilePersistence {
   }
 
   public async notify(currentAssets: BotAssets, previousAssets: BotAssets) {
-    const fileChanges: IFileChange[] = this.getAssetsChanges(currentAssets, previousAssets);
+    const fileChanges: IFileChange[] = await this.getAssetsChanges(currentAssets, previousAssets);
+
+    this.createTaskQueue(fileChanges);
 
+    await this.flush();
+  }
+
+  public createTaskQueue(fileChanges: IFileChange[]) {
     for (const change of fileChanges) {
       if (!this._taskQueue[change.id]) {
         this._taskQueue[change.id] = [];
       }
       this._taskQueue[change.id].push(change);
     }
-
-    await this.flush();
   }
 
   public async flush(): Promise<boolean> {
@@ -136,75 +127,15 @@ class FilePersistence {
     return { id: `${file.id}${fileExtension}`, change: content, type: changeType, projectId: this._projectId };
   }
 
-  private getDifferenceItems(current: any[], previous: any[]) {
-    const changes1 = differenceWith(current, previous, isEqual);
-    const changes2 = differenceWith(previous, current, isEqual);
-    const deleted = changes2.filter((change) => !current.some((file) => change.id === file.id));
-    const { updated, added } = changes1.reduce(
-      (result: { updated: any[]; added: any[] }, change) => {
-        if (previous.some((file) => change.id === file.id)) {
-          result.updated.push(change);
-        } else {
-          result.added.push(change);
-        }
-
-        return result;
-      },
-      { updated: [], added: [] }
-    );
-
-    return { updated, added, deleted };
-  }
-
-  private getFileChanges(fileExtension: FileExtensions, changes: { updated: any[]; added: any[]; deleted: any[] }) {
+  private async getFilesChanges(current: FileAsset[], previous: FileAsset[], fileExtension: FileExtensions) {
+    const changes = (await fileDiffCalculator.difference(current, previous)) as FileDifference;
     const updated = changes.updated.map((file) => this.createChange(file, fileExtension, ChangeType.UPDATE));
     const added = changes.added.map((file) => this.createChange(file, fileExtension, ChangeType.CREATE));
     const deleted = changes.deleted.map((file) => this.createChange(file, fileExtension, ChangeType.DELETE));
     const fileChanges: IFileChange[] = [...updated, ...added, ...deleted];
     return fileChanges;
   }
 
-  private getDialogChanges(current: DialogInfo[], previous: DialogInfo[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Dialog, changeItems);
-    return changes;
-  }
-
-  private getDialogSchemaChanges(current: DialogSchemaFile[], previous: DialogSchemaFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    return this.getFileChanges(FileExtensions.DialogSchema, changeItems);
-  }
-
-  private getLuChanges(current: LuFile[], previous: LuFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Lu, changeItems);
-    return changes;
-  }
-
-  private getQnAChanges(current: QnAFile[], previous: QnAFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.QnA, changeItems);
-    return changes;
-  }
-
-  private getLgChanges(current: LgFile[], previous: LgFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Lg, changeItems);
-    return changes;
-  }
-
-  private getSkillManifestsChanges(current: SkillManifestFile[], previous: SkillManifestFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Manifest, changeItems);
-    return changes;
-  }
-
-  private getRecognizerChanges(current: RecognizerFile[], previous: RecognizerFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Recognizer, changeItems);
-    return changes;
-  }
-
   private getCrossTrainConfigChanges(current: CrosstrainConfig, previous: CrosstrainConfig) {
     if (isEqual(current, previous)) return [];
     let changeType = ChangeType.UPDATE;
@@ -245,54 +176,38 @@ class FilePersistence {
     return [];
   }
 
-  private getFormDialogSchemaFileChanges(current: FormDialogSchema[], previous: FormDialogSchema[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.FormDialogSchema, changeItems);
-    return changes;
-  }
+  public async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise<IFileChange[]> {
+    const files: [FileAsset[], FileAsset[], FileExtensions][] = [
+      [currentAssets.dialogs, previousAssets.dialogs, FileExtensions.Dialog],
+      [currentAssets.dialogSchemas, previousAssets.dialogSchemas, FileExtensions.DialogSchema],
+      [currentAssets.luFiles, previousAssets.luFiles, FileExtensions.Lu],
+      [currentAssets.qnaFiles, previousAssets.qnaFiles, FileExtensions.QnA],
+      [currentAssets.lgFiles, previousAssets.lgFiles, FileExtensions.Lg],
+      [currentAssets.skillManifests, previousAssets.skillManifests, FileExtensions.Manifest],
+      [currentAssets.formDialogSchemas, previousAssets.formDialogSchemas, FileExtensions.FormDialogSchema],
+      [currentAssets.recognizers, previousAssets.recognizers, FileExtensions.Recognizer],
+    ];
 
-  private getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): IFileChange[] {
-    const dialogChanges = this.getDialogChanges(currentAssets.dialogs, previousAssets.dialogs);
-    const dialogSchemaChanges = this.getDialogSchemaChanges(currentAssets.dialogSchemas, previousAssets.dialogSchemas);
-    const luChanges = this.getLuChanges(currentAssets.luFiles, previousAssets.luFiles);
-    const qnaChanges = this.getQnAChanges(currentAssets.qnaFiles, previousAssets.qnaFiles);
-    const lgChanges = this.getLgChanges(currentAssets.lgFiles, previousAssets.lgFiles);
-    const skillManifestChanges = this.getSkillManifestsChanges(
-      currentAssets.skillManifests,
-      previousAssets.skillManifests
+    const changes = await Promise.all(
+      files.map(async (item) => {
+        return await this.getFilesChanges(item[0], item[1], item[2]);
+      })
     );
-    const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting);
 
-    const formDialogChanges = this.getFormDialogSchemaFileChanges(
-      currentAssets.formDialogSchemas,
-      previousAssets.formDialogSchemas
-    );
+    const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting);
 
     const botProjectFileChanges = this.getBotProjectFileChanges(
       currentAssets.botProjectFile,
       previousAssets.botProjectFile
     );
 
-    const recognizerFileChanges = this.getRecognizerChanges(currentAssets.recognizers, previousAssets.recognizers);
-
     const crossTrainFileChanges = this.getCrossTrainConfigChanges(
       currentAssets.crossTrainConfig,
       previousAssets.crossTrainConfig
     );
 
-    const fileChanges: IFileChange[] = [
-      ...dialogChanges,
-      ...dialogSchemaChanges,
-      ...luChanges,
-      ...qnaChanges,
-      ...lgChanges,
-      ...skillManifestChanges,
-      ...settingChanges,
-      ...formDialogChanges,
-      ...botProjectFileChanges,
-      ...recognizerFileChanges,
-      ...crossTrainFileChanges,
-    ];
+    const fileChanges: IFileChange[] = [...settingChanges, ...botProjectFileChanges, ...crossTrainFileChanges];
+    changes.forEach((item) => fileChanges.push(...item));
     return fileChanges;
   }
 }