fix: Move persistence layer's delta computation into worker

Composer/packages/client/src/recoilModel/parsers/calculator.ts

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { FileAsset } from '../persistence/types';
+
+import Worker from './workers/calculator.worker.ts';
+import { BaseWorker } from './baseWorker';
+import { FilesDifferencePayload, CalculatorType } from './types';
+
+// Wrapper class
+class Calculator extends BaseWorker<CalculatorType> {
+  difference(target: FileAsset[], origin: FileAsset[]) {
+    const payload = { target, origin };
+    return this.sendMsg<FilesDifferencePayload>('difference', payload);
+  }
+}
+
+export default new Calculator(new Worker());

Composer/packages/client/src/recoilModel/parsers/types.ts

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 import { LuIntentSection, LgFile, LuFile, QnASection, FileInfo, LgTemplate, ILUFeaturesConfig } from '@bfc/shared';
 
+import { FileAsset } from '../persistence/types';
+
 export type LuParsePayload = {
   id: string;
   content: string;
@@ -144,3 +146,10 @@ export enum QnAActionType {
   UpdateSection = 'update-section',
   RemoveSection = 'remove-section',
 }
+
+export type FilesDifferencePayload = {
+  target: FileAsset[];
+  origin: FileAsset[];
+};
+
+export type CalculatorType = 'difference';

Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import isEqual from 'lodash/isEqual';
+import differenceWith from 'lodash/differenceWith';
+
+import { FileAsset } from '../../persistence/types';
+
+import { CalculatorType, FilesDifferencePayload } from './../types';
+
+type DifferenceMessage = {
+  id: string;
+  type: CalculatorType;
+  payload: FilesDifferencePayload;
+};
+
+type MessageEvent = DifferenceMessage;
+
+const ctx: Worker = self as any;
+
+export function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) {
+  const changes1 = differenceWith(target, origin, isEqual);
+  const changes2 = differenceWith(origin, target, isEqual);
+  const deleted = changes2.filter((change) => !target.some((file) => change.id === file.id));
+  const { updated, added } = changes1.reduce(
+    (result: { updated: FileAsset[]; added: FileAsset[] }, change) => {
+      if (origin.some((file) => change.id === file.id)) {
+        result.updated.push(change);
+      } else {
+        result.added.push(change);
+      }
+
+      return result;
+    },
+    { updated: [], added: [] }
+  );
+
+  return { updated, added, deleted };
+}
+
+function handleMessage(message: MessageEvent) {
+  const { type, payload } = message;
+
+  switch (type) {
+    case 'difference': {
+      const { target, origin } = payload;
+      return getDifferenceItems(target, origin);
+    }
+    default: {
+      return null;
+    }
+  }
+}
+
+ctx.onmessage = function (event) {
+  const msg = event.data as MessageEvent;
+  try {
+    const result = handleMessage(msg);
+    ctx.postMessage({ id: msg.id, payload: result });
+  } catch (error) {
+    ctx.postMessage({ id: msg.id, error: error.message });
+  }
+};

Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts

@@ -1,26 +1,12 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-
-import differenceWith from 'lodash/differenceWith';
 import isEqual from 'lodash/isEqual';
-import {
-  DialogInfo,
-  DialogSchemaFile,
-  DialogSetting,
-  BotAssets,
-  BotProjectFile,
-  LuFile,
-  LgFile,
-  QnAFile,
-  FormDialogSchema,
-  RecognizerFile,
-  CrosstrainConfig,
-  SkillManifestFile,
-} from '@bfc/shared';
+import { DialogSetting, BotAssets, BotProjectFile, CrosstrainConfig } from '@bfc/shared';
 import keys from 'lodash/keys';
 
 import * as client from './http';
-import { ChangeType, FileExtensions, IFileChange } from './types';
+import calculator from './../parsers/calculator';
+import { ChangeType, Difference, FileExtensions, IFileChange, FileAsset } from './types';
 
 class FilePersistence {
   private _taskQueue: { [id: string]: IFileChange[] } = {};
@@ -47,16 +33,20 @@ class FilePersistence {
   }
 
   public async notify(currentAssets: BotAssets, previousAssets: BotAssets) {
-    const fileChanges: IFileChange[] = this.getAssetsChanges(currentAssets, previousAssets);
+    const fileChanges: IFileChange[] = await this.getAssetsChanges(currentAssets, previousAssets);
+
+    this.createTaskQueue(fileChanges);
+
+    await this.flush();
+  }
 
+  public createTaskQueue(fileChanges: IFileChange[]) {
     for (const change of fileChanges) {
       if (!this._taskQueue[change.id]) {
         this._taskQueue[change.id] = [];
       }
       this._taskQueue[change.id].push(change);
     }
-
-    await this.flush();
   }
 
   public async flush(): Promise<boolean> {
@@ -136,75 +126,15 @@ class FilePersistence {
     return { id: `${file.id}${fileExtension}`, change: content, type: changeType, projectId: this._projectId };
   }
 
-  private getDifferenceItems(current: any[], previous: any[]) {
-    const changes1 = differenceWith(current, previous, isEqual);
-    const changes2 = differenceWith(previous, current, isEqual);
-    const deleted = changes2.filter((change) => !current.some((file) => change.id === file.id));
-    const { updated, added } = changes1.reduce(
-      (result: { updated: any[]; added: any[] }, change) => {
-        if (previous.some((file) => change.id === file.id)) {
-          result.updated.push(change);
-        } else {
-          result.added.push(change);
-        }
-
-        return result;
-      },
-      { updated: [], added: [] }
-    );
-
-    return { updated, added, deleted };
-  }
-
-  private getFileChanges(fileExtension: FileExtensions, changes: { updated: any[]; added: any[]; deleted: any[] }) {
+  private async getFilesChanges(current: FileAsset[], previous: FileAsset[], fileExtension: FileExtensions) {
+    const changes = (await calculator.difference(current, previous)) as Difference;
     const updated = changes.updated.map((file) => this.createChange(file, fileExtension, ChangeType.UPDATE));
     const added = changes.added.map((file) => this.createChange(file, fileExtension, ChangeType.CREATE));
     const deleted = changes.deleted.map((file) => this.createChange(file, fileExtension, ChangeType.DELETE));
     const fileChanges: IFileChange[] = [...updated, ...added, ...deleted];
     return fileChanges;
   }
 
-  private getDialogChanges(current: DialogInfo[], previous: DialogInfo[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Dialog, changeItems);
-    return changes;
-  }
-
-  private getDialogSchemaChanges(current: DialogSchemaFile[], previous: DialogSchemaFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    return this.getFileChanges(FileExtensions.DialogSchema, changeItems);
-  }
-
-  private getLuChanges(current: LuFile[], previous: LuFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Lu, changeItems);
-    return changes;
-  }
-
-  private getQnAChanges(current: QnAFile[], previous: QnAFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.QnA, changeItems);
-    return changes;
-  }
-
-  private getLgChanges(current: LgFile[], previous: LgFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Lg, changeItems);
-    return changes;
-  }
-
-  private getSkillManifestsChanges(current: SkillManifestFile[], previous: SkillManifestFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Manifest, changeItems);
-    return changes;
-  }
-
-  private getRecognizerChanges(current: RecognizerFile[], previous: RecognizerFile[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.Recognizer, changeItems);
-    return changes;
-  }
-
   private getCrossTrainConfigChanges(current: CrosstrainConfig, previous: CrosstrainConfig) {
     if (isEqual(current, previous)) return [];
     let changeType = ChangeType.UPDATE;
@@ -245,54 +175,38 @@ class FilePersistence {
     return [];
   }
 
-  private getFormDialogSchemaFileChanges(current: FormDialogSchema[], previous: FormDialogSchema[]) {
-    const changeItems = this.getDifferenceItems(current, previous);
-    const changes = this.getFileChanges(FileExtensions.FormDialogSchema, changeItems);
-    return changes;
-  }
+  public async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise<IFileChange[]> {
+    const files: [FileAsset[], FileAsset[], FileExtensions][] = [
+      [currentAssets.dialogs, previousAssets.dialogs, FileExtensions.Dialog],
+      [currentAssets.dialogSchemas, previousAssets.dialogSchemas, FileExtensions.DialogSchema],
+      [currentAssets.luFiles, previousAssets.luFiles, FileExtensions.Lu],
+      [currentAssets.qnaFiles, previousAssets.qnaFiles, FileExtensions.QnA],
+      [currentAssets.lgFiles, previousAssets.lgFiles, FileExtensions.Lg],
+      [currentAssets.skillManifests, previousAssets.skillManifests, FileExtensions.Manifest],
+      [currentAssets.formDialogSchemas, previousAssets.formDialogSchemas, FileExtensions.FormDialogSchema],
+      [currentAssets.recognizers, previousAssets.recognizers, FileExtensions.Recognizer],
+    ];
 
-  private getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): IFileChange[] {
-    const dialogChanges = this.getDialogChanges(currentAssets.dialogs, previousAssets.dialogs);
-    const dialogSchemaChanges = this.getDialogSchemaChanges(currentAssets.dialogSchemas, previousAssets.dialogSchemas);
-    const luChanges = this.getLuChanges(currentAssets.luFiles, previousAssets.luFiles);
-    const qnaChanges = this.getQnAChanges(currentAssets.qnaFiles, previousAssets.qnaFiles);
-    const lgChanges = this.getLgChanges(currentAssets.lgFiles, previousAssets.lgFiles);
-    const skillManifestChanges = this.getSkillManifestsChanges(
-      currentAssets.skillManifests,
-      previousAssets.skillManifests
+    const changes = await Promise.all(
+      files.map(async (item) => {
+        return await this.getFilesChanges(item[0], item[1], item[2]);
+      })
     );
-    const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting);
 
-    const formDialogChanges = this.getFormDialogSchemaFileChanges(
-      currentAssets.formDialogSchemas,
-      previousAssets.formDialogSchemas
-    );
+    const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting);
 
     const botProjectFileChanges = this.getBotProjectFileChanges(
       currentAssets.botProjectFile,
       previousAssets.botProjectFile
     );
 
-    const recognizerFileChanges = this.getRecognizerChanges(currentAssets.recognizers, previousAssets.recognizers);
-
     const crossTrainFileChanges = this.getCrossTrainConfigChanges(
       currentAssets.crossTrainConfig,
       previousAssets.crossTrainConfig
     );
 
-    const fileChanges: IFileChange[] = [
-      ...dialogChanges,
-      ...dialogSchemaChanges,
-      ...luChanges,
-      ...qnaChanges,
-      ...lgChanges,
-      ...skillManifestChanges,
-      ...settingChanges,
-      ...formDialogChanges,
-      ...botProjectFileChanges,
-      ...recognizerFileChanges,
-      ...crossTrainFileChanges,
-    ];
+    const fileChanges: IFileChange[] = [...settingChanges, ...botProjectFileChanges, ...crossTrainFileChanges];
+    changes.forEach((item) => fileChanges.push(...item));
     return fileChanges;
   }
 }

Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts

@@ -17,6 +17,12 @@ jest.mock('axios', () => {
   };
 });
 
+jest.mock('../../parsers/calculator', () => {
+  return {
+    difference: require('../../parsers/workers/calculator.worker').getDifferenceItems,
+  };
+});
+
 describe('test persistence layer', () => {
   let filePersistence: FilePersistence;
   beforeEach(() => {
@@ -26,26 +32,42 @@ describe('test persistence layer', () => {
   it('test notify update', async () => {
     const previous = {
       projectId: 'test',
-      dialogs: ([{ id: 'a', content: { a: 'old' } }] as unknown) as DialogInfo[],
+      dialogs: [] as DialogInfo[],
       dialogSchemas: [{ id: 'a', content: { a: 'old schema' } }] as DialogSchemaFile[],
       lgFiles: [{ id: 'a.en-us', content: '' }] as LgFile[],
       luFiles: [{ id: 'a.en-us', content: '' }] as LuFile[],
+      crossTrainConfig: { 'cross-train.config.json': { rootBot: false } } as any,
     } as BotAssets;
 
     const current = {
       projectId: 'test',
-      dialogs: ([{ id: 'a', content: { a: 'new' } }] as unknown) as DialogInfo[],
+      dialogs: ([{ id: 'a', content: { a: 'create' } }] as unknown) as DialogInfo[],
+      dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[],
+      lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[],
+      luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[],
+      crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any,
+    } as BotAssets;
+
+    const last = {
+      projectId: 'test',
+      dialogs: ([{ id: 'a', content: { a: 'update' } }] as unknown) as DialogInfo[],
       dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[],
       lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[],
       luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[],
+      crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any,
     } as BotAssets;
 
-    filePersistence.notify(current, previous);
-    filePersistence.notify(current, previous);
-    expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('new');
+    const result = await filePersistence.getAssetsChanges(current, previous);
+    filePersistence.createTaskQueue(result);
+    expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('create');
     expect(JSON.parse(filePersistence.taskQueue['a.dialog.schema'][0].change).a).toBe('new schema');
     expect(filePersistence.taskQueue['a.en-us.lg'][0].change).toBe('a.lg');
     expect(filePersistence.taskQueue['a.en-us.lu'][0].change).toBe('a.lu');
+    const result1 = await filePersistence.getAssetsChanges(last, current);
+    filePersistence.createTaskQueue(result1);
+    filePersistence.flush();
+    filePersistence.flush();
+    expect(filePersistence.taskQueue['a.en-us.lu'].length).toBe(0);
   });
 
   it('test notify create', async () => {
@@ -77,12 +99,14 @@ describe('test persistence layer', () => {
       ] as LuFile[],
     } as BotAssets;
 
-    filePersistence.notify(current, previous);
-    filePersistence.notify(current, previous);
+    const result = await filePersistence.getAssetsChanges(current, previous);
+    filePersistence.createTaskQueue(result);
     expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b');
     expect(JSON.parse(filePersistence.taskQueue['b.dialog.schema'][0].change).b).toBe('b');
     expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.lg');
     expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.lu');
+    await filePersistence.flush();
+    expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0);
   });
 
   it('test notify remove', async () => {
@@ -113,10 +137,12 @@ describe('test persistence layer', () => {
       lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[],
       luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[],
     } as BotAssets;
-    filePersistence.notify(current, previous);
-    filePersistence.notify(current, previous);
+    const result = await filePersistence.getAssetsChanges(current, previous);
+    filePersistence.createTaskQueue(result);
     expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b.pre');
     expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.pre.lg');
     expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.pre.lu');
+    await filePersistence.flush();
+    expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0);
   });
 });