diff --git a/Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts b/Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts new file mode 100644 index 0000000000..e778cfbc7a --- /dev/null +++ b/Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { FileAsset } from '../persistence/types'; + +import Worker from './workers/calculator.worker.ts'; +import { BaseWorker } from './baseWorker'; +import { FilesDifferencePayload, CalculatorType } from './types'; + +// Wrapper class +class FileDiffCalculator extends BaseWorker { + difference(target: FileAsset[], origin: FileAsset[]) { + const payload = { target, origin }; + return this.sendMsg('difference', payload); + } +} + +export default new FileDiffCalculator(new Worker()); diff --git a/Composer/packages/client/src/recoilModel/parsers/types.ts b/Composer/packages/client/src/recoilModel/parsers/types.ts index a5e0c484fb..77311b6dfd 100644 --- a/Composer/packages/client/src/recoilModel/parsers/types.ts +++ b/Composer/packages/client/src/recoilModel/parsers/types.ts @@ -2,6 +2,8 @@ // Licensed under the MIT License. import { LuIntentSection, LgFile, LuFile, QnASection, FileInfo, LgTemplate, ILUFeaturesConfig } from '@bfc/shared'; +import { FileAsset } from '../persistence/types'; + export type LuParsePayload = { id: string; content: string; @@ -144,3 +146,10 @@ export enum QnAActionType { UpdateSection = 'update-section', RemoveSection = 'remove-section', } + +export type FilesDifferencePayload = { + target: FileAsset[]; + origin: FileAsset[]; +}; + +export type CalculatorType = 'difference'; diff --git a/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts new file mode 100644 index 0000000000..610da18579 --- /dev/null +++ b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import isEqual from 'lodash/isEqual'; +import differenceWith from 'lodash/differenceWith'; + +import { FileAsset } from '../../persistence/types'; + +import { CalculatorType, FilesDifferencePayload } from './../types'; + +type DifferenceMessage = { + id: string; + type: CalculatorType; + payload: FilesDifferencePayload; +}; + +type MessageEvent = DifferenceMessage; + +const ctx: Worker = self as any; + +export function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) { + const changes1 = differenceWith(target, origin, isEqual); + const changes2 = differenceWith(origin, target, isEqual); + const deleted = changes2.filter((change) => !target.some((file) => change.id === file.id)); + const { updated, added } = changes1.reduce( + (result: { updated: FileAsset[]; added: FileAsset[] }, change) => { + if (origin.some((file) => change.id === file.id)) { + result.updated.push(change); + } else { + result.added.push(change); + } + + return result; + }, + { updated: [], added: [] } + ); + + return { updated, added, deleted }; +} + +function handleMessage(message: MessageEvent) { + const { type, payload } = message; + + switch (type) { + case 'difference': { + const { target, origin } = payload; + return getDifferenceItems(target, origin); + } + default: { + return null; + } + } +} + +ctx.onmessage = function (event) { + const msg = event.data as MessageEvent; + try { + const result = handleMessage(msg); + ctx.postMessage({ id: msg.id, payload: result }); + } catch (error) { + ctx.postMessage({ id: msg.id, error: error.message }); + } +}; diff --git a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts index e7f0b64728..90ec5d7b6d 100644 --- a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts +++ b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts @@ -1,26 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. - -import differenceWith from 'lodash/differenceWith'; import isEqual from 'lodash/isEqual'; -import { - DialogInfo, - DialogSchemaFile, - DialogSetting, - BotAssets, - BotProjectFile, - LuFile, - LgFile, - QnAFile, - FormDialogSchema, - RecognizerFile, - CrosstrainConfig, - SkillManifestFile, -} from '@bfc/shared'; +import { DialogSetting, BotAssets, BotProjectFile, CrosstrainConfig } from '@bfc/shared'; import keys from 'lodash/keys'; +import fileDiffCalculator from '../parsers/fileDiffCalculator'; + import * as client from './http'; -import { ChangeType, FileExtensions, IFileChange } from './types'; +import { ChangeType, FileDifference, FileExtensions, IFileChange, FileAsset } from './types'; class FilePersistence { private _taskQueue: { [id: string]: IFileChange[] } = {}; @@ -47,16 +34,20 @@ class FilePersistence { } public async notify(currentAssets: BotAssets, previousAssets: BotAssets) { - const fileChanges: IFileChange[] = this.getAssetsChanges(currentAssets, previousAssets); + const fileChanges: IFileChange[] = await this.getAssetsChanges(currentAssets, previousAssets); + + this.createTaskQueue(fileChanges); + await this.flush(); + } + + public createTaskQueue(fileChanges: IFileChange[]) { for (const change of fileChanges) { if (!this._taskQueue[change.id]) { this._taskQueue[change.id] = []; } this._taskQueue[change.id].push(change); } - - await this.flush(); } public async flush(): Promise { @@ -136,27 +127,8 @@ class FilePersistence { return { id: `${file.id}${fileExtension}`, change: content, type: changeType, projectId: this._projectId }; } - private getDifferenceItems(current: any[], previous: any[]) { - const changes1 = differenceWith(current, previous, isEqual); - const changes2 = differenceWith(previous, current, isEqual); - const deleted = changes2.filter((change) => !current.some((file) => change.id === file.id)); - const { updated, added } = changes1.reduce( - (result: { updated: any[]; added: any[] }, change) => { - if (previous.some((file) => change.id === file.id)) { - result.updated.push(change); - } else { - result.added.push(change); - } - - return result; - }, - { updated: [], added: [] } - ); - - return { updated, added, deleted }; - } - - private getFileChanges(fileExtension: FileExtensions, changes: { updated: any[]; added: any[]; deleted: any[] }) { + private async getFilesChanges(current: FileAsset[], previous: FileAsset[], fileExtension: FileExtensions) { + const changes = (await fileDiffCalculator.difference(current, previous)) as FileDifference; const updated = changes.updated.map((file) => this.createChange(file, fileExtension, ChangeType.UPDATE)); const added = changes.added.map((file) => this.createChange(file, fileExtension, ChangeType.CREATE)); const deleted = changes.deleted.map((file) => this.createChange(file, fileExtension, ChangeType.DELETE)); @@ -164,47 +136,6 @@ class FilePersistence { return fileChanges; } - private getDialogChanges(current: DialogInfo[], previous: DialogInfo[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Dialog, changeItems); - return changes; - } - - private getDialogSchemaChanges(current: DialogSchemaFile[], previous: DialogSchemaFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - return this.getFileChanges(FileExtensions.DialogSchema, changeItems); - } - - private getLuChanges(current: LuFile[], previous: LuFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Lu, changeItems); - return changes; - } - - private getQnAChanges(current: QnAFile[], previous: QnAFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.QnA, changeItems); - return changes; - } - - private getLgChanges(current: LgFile[], previous: LgFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Lg, changeItems); - return changes; - } - - private getSkillManifestsChanges(current: SkillManifestFile[], previous: SkillManifestFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Manifest, changeItems); - return changes; - } - - private getRecognizerChanges(current: RecognizerFile[], previous: RecognizerFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Recognizer, changeItems); - return changes; - } - private getCrossTrainConfigChanges(current: CrosstrainConfig, previous: CrosstrainConfig) { if (isEqual(current, previous)) return []; let changeType = ChangeType.UPDATE; @@ -245,54 +176,38 @@ class FilePersistence { return []; } - private getFormDialogSchemaFileChanges(current: FormDialogSchema[], previous: FormDialogSchema[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.FormDialogSchema, changeItems); - return changes; - } + public async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise { + const files: [FileAsset[], FileAsset[], FileExtensions][] = [ + [currentAssets.dialogs, previousAssets.dialogs, FileExtensions.Dialog], + [currentAssets.dialogSchemas, previousAssets.dialogSchemas, FileExtensions.DialogSchema], + [currentAssets.luFiles, previousAssets.luFiles, FileExtensions.Lu], + [currentAssets.qnaFiles, previousAssets.qnaFiles, FileExtensions.QnA], + [currentAssets.lgFiles, previousAssets.lgFiles, FileExtensions.Lg], + [currentAssets.skillManifests, previousAssets.skillManifests, FileExtensions.Manifest], + [currentAssets.formDialogSchemas, previousAssets.formDialogSchemas, FileExtensions.FormDialogSchema], + [currentAssets.recognizers, previousAssets.recognizers, FileExtensions.Recognizer], + ]; - private getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): IFileChange[] { - const dialogChanges = this.getDialogChanges(currentAssets.dialogs, previousAssets.dialogs); - const dialogSchemaChanges = this.getDialogSchemaChanges(currentAssets.dialogSchemas, previousAssets.dialogSchemas); - const luChanges = this.getLuChanges(currentAssets.luFiles, previousAssets.luFiles); - const qnaChanges = this.getQnAChanges(currentAssets.qnaFiles, previousAssets.qnaFiles); - const lgChanges = this.getLgChanges(currentAssets.lgFiles, previousAssets.lgFiles); - const skillManifestChanges = this.getSkillManifestsChanges( - currentAssets.skillManifests, - previousAssets.skillManifests + const changes = await Promise.all( + files.map(async (item) => { + return await this.getFilesChanges(item[0], item[1], item[2]); + }) ); - const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting); - const formDialogChanges = this.getFormDialogSchemaFileChanges( - currentAssets.formDialogSchemas, - previousAssets.formDialogSchemas - ); + const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting); const botProjectFileChanges = this.getBotProjectFileChanges( currentAssets.botProjectFile, previousAssets.botProjectFile ); - const recognizerFileChanges = this.getRecognizerChanges(currentAssets.recognizers, previousAssets.recognizers); - const crossTrainFileChanges = this.getCrossTrainConfigChanges( currentAssets.crossTrainConfig, previousAssets.crossTrainConfig ); - const fileChanges: IFileChange[] = [ - ...dialogChanges, - ...dialogSchemaChanges, - ...luChanges, - ...qnaChanges, - ...lgChanges, - ...skillManifestChanges, - ...settingChanges, - ...formDialogChanges, - ...botProjectFileChanges, - ...recognizerFileChanges, - ...crossTrainFileChanges, - ]; + const fileChanges: IFileChange[] = [...settingChanges, ...botProjectFileChanges, ...crossTrainFileChanges]; + changes.forEach((item) => fileChanges.push(...item)); return fileChanges; } } diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index 854b2a7591..7622939471 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -17,6 +17,12 @@ jest.mock('axios', () => { }; }); +jest.mock('../../parsers/fileDiffCalculator', () => { + return { + difference: require('../../parsers/workers/calculator.worker').getDifferenceItems, + }; +}); + describe('test persistence layer', () => { let filePersistence: FilePersistence; beforeEach(() => { @@ -26,26 +32,42 @@ describe('test persistence layer', () => { it('test notify update', async () => { const previous = { projectId: 'test', - dialogs: ([{ id: 'a', content: { a: 'old' } }] as unknown) as DialogInfo[], + dialogs: [] as DialogInfo[], dialogSchemas: [{ id: 'a', content: { a: 'old schema' } }] as DialogSchemaFile[], lgFiles: [{ id: 'a.en-us', content: '' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: '' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: false } } as any, } as BotAssets; const current = { projectId: 'test', - dialogs: ([{ id: 'a', content: { a: 'new' } }] as unknown) as DialogInfo[], + dialogs: ([{ id: 'a', content: { a: 'create' } }] as unknown) as DialogInfo[], + dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[], + lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[], + luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any, + } as BotAssets; + + const last = { + projectId: 'test', + dialogs: ([{ id: 'a', content: { a: 'update' } }] as unknown) as DialogInfo[], dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[], lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any, } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); - expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('new'); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); + expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('create'); expect(JSON.parse(filePersistence.taskQueue['a.dialog.schema'][0].change).a).toBe('new schema'); expect(filePersistence.taskQueue['a.en-us.lg'][0].change).toBe('a.lg'); expect(filePersistence.taskQueue['a.en-us.lu'][0].change).toBe('a.lu'); + const result1 = await filePersistence.getAssetsChanges(last, current); + filePersistence.createTaskQueue(result1); + filePersistence.flush(); + filePersistence.flush(); + expect(filePersistence.taskQueue['a.en-us.lu'].length).toBe(0); }); it('test notify create', async () => { @@ -77,12 +99,14 @@ describe('test persistence layer', () => { ] as LuFile[], } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b'); expect(JSON.parse(filePersistence.taskQueue['b.dialog.schema'][0].change).b).toBe('b'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.lu'); + await filePersistence.flush(); + expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0); }); it('test notify remove', async () => { @@ -113,10 +137,12 @@ describe('test persistence layer', () => { lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[], } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b.pre'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.pre.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.pre.lu'); + await filePersistence.flush(); + expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0); }); }); diff --git a/Composer/packages/client/src/recoilModel/persistence/types.ts b/Composer/packages/client/src/recoilModel/persistence/types.ts index 329441d251..801b1b0d25 100644 --- a/Composer/packages/client/src/recoilModel/persistence/types.ts +++ b/Composer/packages/client/src/recoilModel/persistence/types.ts @@ -1,6 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { + DialogInfo, + DialogSchemaFile, + FormDialogSchema, + LgFile, + LuFile, + QnAFile, + RecognizerFile, + SkillManifestFile, +} from '@bfc/shared'; + export enum ChangeType { DELETE = 1, UPDATE, @@ -30,3 +41,19 @@ export interface IFileChange { change: string; type: ChangeType; } + +export type FileAsset = + | DialogInfo + | DialogSchemaFile + | LuFile + | QnAFile + | LgFile + | SkillManifestFile + | RecognizerFile + | FormDialogSchema; + +export type FileDifference = { + updated: FileAsset[]; + added: FileAsset[]; + deleted: FileAsset[]; +};