From 18037c3892877a2b8b33ba135b02a825d19ca1ac Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 20 Jan 2021 13:55:37 +0800 Subject: [PATCH 1/6] fix: Move persistence layer's delta computation into worker --- .../src/recoilModel/parsers/calculator.ts | 18 +++ .../client/src/recoilModel/parsers/types.ts | 25 +++- .../parsers/workers/calculator.worker.ts | 62 ++++++++ .../persistence/FilePersistence.ts | 139 +++--------------- .../src/recoilModel/persistence/types.ts | 27 ++++ 5 files changed, 155 insertions(+), 116 deletions(-) create mode 100644 Composer/packages/client/src/recoilModel/parsers/calculator.ts create mode 100644 Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts diff --git a/Composer/packages/client/src/recoilModel/parsers/calculator.ts b/Composer/packages/client/src/recoilModel/parsers/calculator.ts new file mode 100644 index 0000000000..04cc6417ba --- /dev/null +++ b/Composer/packages/client/src/recoilModel/parsers/calculator.ts @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { FileAsset } from '../persistence/types'; + +import Worker from './workers/calculator.worker.ts'; +import { BaseWorker } from './baseWorker'; +import { FilesDifferencePayload, CalculatorType } from './types'; + +// Wrapper class +class Calculator extends BaseWorker { + difference(target: FileAsset[], origin: FileAsset[]) { + const payload = { target, origin }; + return this.sendMsg('difference', payload); + } +} + +export default new Calculator(new Worker()); diff --git a/Composer/packages/client/src/recoilModel/parsers/types.ts b/Composer/packages/client/src/recoilModel/parsers/types.ts index a5e0c484fb..61b36fd388 100644 --- a/Composer/packages/client/src/recoilModel/parsers/types.ts +++ b/Composer/packages/client/src/recoilModel/parsers/types.ts @@ -1,6 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { LuIntentSection, LgFile, LuFile, QnASection, FileInfo, LgTemplate, ILUFeaturesConfig } from '@bfc/shared'; +import { + LuIntentSection, + LgFile, + LuFile, + QnASection, + FileInfo, + LgTemplate, + ILUFeaturesConfig, + DialogInfo, + DialogSchemaFile, + QnAFile, + SkillManifestFile, + RecognizerFile, + FormDialogSchema, +} from '@bfc/shared'; + +import { FileAsset } from '../persistence/types'; export type LuParsePayload = { id: string; @@ -144,3 +160,10 @@ export enum QnAActionType { UpdateSection = 'update-section', RemoveSection = 'remove-section', } + +export type FilesDifferencePayload = { + target: FileAsset[]; + origin: FileAsset[]; +}; + +export type CalculatorType = 'difference'; diff --git a/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts new file mode 100644 index 0000000000..34589e680a --- /dev/null +++ b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import isEqual from 'lodash/isEqual'; +import differenceWith from 'lodash/differenceWith'; + +import { FileAsset } from '../../persistence/types'; + +import { CalculatorType, FilesDifferencePayload } from './../types'; + +type DifferenceMessage = { + id: string; + type: CalculatorType; + payload: FilesDifferencePayload; +}; + +type MessageEvent = DifferenceMessage; + +const ctx: Worker = self as any; + +function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) { + const changes1 = differenceWith(target, origin, isEqual); + const changes2 = differenceWith(origin, target, isEqual); + const deleted = changes2.filter((change) => !target.some((file) => change.id === file.id)); + const { updated, added } = changes1.reduce( + (result: { updated: FileAsset[]; added: FileAsset[] }, change) => { + if (origin.some((file) => change.id === file.id)) { + result.updated.push(change); + } else { + result.added.push(change); + } + + return result; + }, + { updated: [], added: [] } + ); + + return { updated, added, deleted }; +} + +function handleMessage(message: MessageEvent) { + const { type, payload } = message; + + switch (type) { + case 'difference': { + const { target, origin } = payload; + return getDifferenceItems(target, origin); + } + default: { + return null; + } + } +} + +ctx.onmessage = function (event) { + const msg = event.data as MessageEvent; + try { + const result = handleMessage(msg); + ctx.postMessage({ id: msg.id, payload: result }); + } catch (error) { + ctx.postMessage({ id: msg.id, error: error.message }); + } +}; diff --git a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts index e7f0b64728..253f293562 100644 --- a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts +++ b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts @@ -1,26 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. - -import differenceWith from 'lodash/differenceWith'; import isEqual from 'lodash/isEqual'; -import { - DialogInfo, - DialogSchemaFile, - DialogSetting, - BotAssets, - BotProjectFile, - LuFile, - LgFile, - QnAFile, - FormDialogSchema, - RecognizerFile, - CrosstrainConfig, - SkillManifestFile, -} from '@bfc/shared'; +import { DialogSetting, BotAssets, BotProjectFile, CrosstrainConfig } from '@bfc/shared'; import keys from 'lodash/keys'; import * as client from './http'; -import { ChangeType, FileExtensions, IFileChange } from './types'; +import calculator from './../parsers/calculator'; +import { ChangeType, Difference, FileExtensions, IFileChange, FileAsset } from './types'; class FilePersistence { private _taskQueue: { [id: string]: IFileChange[] } = {}; @@ -47,7 +33,7 @@ class FilePersistence { } public async notify(currentAssets: BotAssets, previousAssets: BotAssets) { - const fileChanges: IFileChange[] = this.getAssetsChanges(currentAssets, previousAssets); + const fileChanges: IFileChange[] = await this.getAssetsChanges(currentAssets, previousAssets); for (const change of fileChanges) { if (!this._taskQueue[change.id]) { @@ -55,7 +41,6 @@ class FilePersistence { } this._taskQueue[change.id].push(change); } - await this.flush(); } @@ -136,27 +121,8 @@ class FilePersistence { return { id: `${file.id}${fileExtension}`, change: content, type: changeType, projectId: this._projectId }; } - private getDifferenceItems(current: any[], previous: any[]) { - const changes1 = differenceWith(current, previous, isEqual); - const changes2 = differenceWith(previous, current, isEqual); - const deleted = changes2.filter((change) => !current.some((file) => change.id === file.id)); - const { updated, added } = changes1.reduce( - (result: { updated: any[]; added: any[] }, change) => { - if (previous.some((file) => change.id === file.id)) { - result.updated.push(change); - } else { - result.added.push(change); - } - - return result; - }, - { updated: [], added: [] } - ); - - return { updated, added, deleted }; - } - - private getFileChanges(fileExtension: FileExtensions, changes: { updated: any[]; added: any[]; deleted: any[] }) { + private async getFilesChanges(current: FileAsset[], previous: FileAsset[], fileExtension: FileExtensions) { + const changes = (await calculator.difference(current, previous)) as Difference; const updated = changes.updated.map((file) => this.createChange(file, fileExtension, ChangeType.UPDATE)); const added = changes.added.map((file) => this.createChange(file, fileExtension, ChangeType.CREATE)); const deleted = changes.deleted.map((file) => this.createChange(file, fileExtension, ChangeType.DELETE)); @@ -164,47 +130,6 @@ class FilePersistence { return fileChanges; } - private getDialogChanges(current: DialogInfo[], previous: DialogInfo[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Dialog, changeItems); - return changes; - } - - private getDialogSchemaChanges(current: DialogSchemaFile[], previous: DialogSchemaFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - return this.getFileChanges(FileExtensions.DialogSchema, changeItems); - } - - private getLuChanges(current: LuFile[], previous: LuFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Lu, changeItems); - return changes; - } - - private getQnAChanges(current: QnAFile[], previous: QnAFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.QnA, changeItems); - return changes; - } - - private getLgChanges(current: LgFile[], previous: LgFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Lg, changeItems); - return changes; - } - - private getSkillManifestsChanges(current: SkillManifestFile[], previous: SkillManifestFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Manifest, changeItems); - return changes; - } - - private getRecognizerChanges(current: RecognizerFile[], previous: RecognizerFile[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.Recognizer, changeItems); - return changes; - } - private getCrossTrainConfigChanges(current: CrosstrainConfig, previous: CrosstrainConfig) { if (isEqual(current, previous)) return []; let changeType = ChangeType.UPDATE; @@ -245,54 +170,38 @@ class FilePersistence { return []; } - private getFormDialogSchemaFileChanges(current: FormDialogSchema[], previous: FormDialogSchema[]) { - const changeItems = this.getDifferenceItems(current, previous); - const changes = this.getFileChanges(FileExtensions.FormDialogSchema, changeItems); - return changes; - } + private async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise { + const files: [FileAsset[], FileAsset[], FileExtensions][] = [ + [currentAssets.dialogs, previousAssets.dialogs, FileExtensions.Dialog], + [currentAssets.dialogSchemas, previousAssets.dialogSchemas, FileExtensions.DialogSchema], + [currentAssets.luFiles, previousAssets.luFiles, FileExtensions.Lu], + [currentAssets.qnaFiles, previousAssets.qnaFiles, FileExtensions.QnA], + [currentAssets.lgFiles, previousAssets.lgFiles, FileExtensions.Lg], + [currentAssets.skillManifests, previousAssets.skillManifests, FileExtensions.Manifest], + [currentAssets.formDialogSchemas, previousAssets.formDialogSchemas, FileExtensions.FormDialogSchema], + [currentAssets.recognizers, previousAssets.recognizers, FileExtensions.Recognizer], + ]; - private getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): IFileChange[] { - const dialogChanges = this.getDialogChanges(currentAssets.dialogs, previousAssets.dialogs); - const dialogSchemaChanges = this.getDialogSchemaChanges(currentAssets.dialogSchemas, previousAssets.dialogSchemas); - const luChanges = this.getLuChanges(currentAssets.luFiles, previousAssets.luFiles); - const qnaChanges = this.getQnAChanges(currentAssets.qnaFiles, previousAssets.qnaFiles); - const lgChanges = this.getLgChanges(currentAssets.lgFiles, previousAssets.lgFiles); - const skillManifestChanges = this.getSkillManifestsChanges( - currentAssets.skillManifests, - previousAssets.skillManifests + const changes = await Promise.all( + files.map(async (item) => { + return await this.getFilesChanges(item[0], item[1], item[2]); + }) ); - const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting); - const formDialogChanges = this.getFormDialogSchemaFileChanges( - currentAssets.formDialogSchemas, - previousAssets.formDialogSchemas - ); + const settingChanges = this.getSettingsChanges(currentAssets.setting, previousAssets.setting); const botProjectFileChanges = this.getBotProjectFileChanges( currentAssets.botProjectFile, previousAssets.botProjectFile ); - const recognizerFileChanges = this.getRecognizerChanges(currentAssets.recognizers, previousAssets.recognizers); - const crossTrainFileChanges = this.getCrossTrainConfigChanges( currentAssets.crossTrainConfig, previousAssets.crossTrainConfig ); - const fileChanges: IFileChange[] = [ - ...dialogChanges, - ...dialogSchemaChanges, - ...luChanges, - ...qnaChanges, - ...lgChanges, - ...skillManifestChanges, - ...settingChanges, - ...formDialogChanges, - ...botProjectFileChanges, - ...recognizerFileChanges, - ...crossTrainFileChanges, - ]; + const fileChanges: IFileChange[] = [...settingChanges, ...botProjectFileChanges, ...crossTrainFileChanges]; + changes.forEach((item) => fileChanges.push(...item)); return fileChanges; } } diff --git a/Composer/packages/client/src/recoilModel/persistence/types.ts b/Composer/packages/client/src/recoilModel/persistence/types.ts index 329441d251..6c6f3bdeb8 100644 --- a/Composer/packages/client/src/recoilModel/persistence/types.ts +++ b/Composer/packages/client/src/recoilModel/persistence/types.ts @@ -1,6 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { + DialogInfo, + DialogSchemaFile, + FormDialogSchema, + LgFile, + LuFile, + QnAFile, + RecognizerFile, + SkillManifestFile, +} from '@bfc/shared'; + export enum ChangeType { DELETE = 1, UPDATE, @@ -30,3 +41,19 @@ export interface IFileChange { change: string; type: ChangeType; } + +export type FileAsset = + | DialogInfo + | DialogSchemaFile + | LuFile + | QnAFile + | LgFile + | SkillManifestFile + | RecognizerFile + | FormDialogSchema; + +export type Difference = { + updated: FileAsset[]; + added: FileAsset[]; + deleted: FileAsset[]; +}; From b11ad9e2f0c0ec60a4e17d2c4666807e8c882652 Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 20 Jan 2021 16:26:26 +0800 Subject: [PATCH 2/6] fix unit testsa --- .../client/src/recoilModel/parsers/types.ts | 16 +--------------- .../parsers/workers/calculator.worker.ts | 2 +- .../recoilModel/persistence/FilePersistence.ts | 9 +++++++-- .../__test__/FilePersistence.test.ts | 18 ++++++++++++------ 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Composer/packages/client/src/recoilModel/parsers/types.ts b/Composer/packages/client/src/recoilModel/parsers/types.ts index 61b36fd388..77311b6dfd 100644 --- a/Composer/packages/client/src/recoilModel/parsers/types.ts +++ b/Composer/packages/client/src/recoilModel/parsers/types.ts @@ -1,20 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { - LuIntentSection, - LgFile, - LuFile, - QnASection, - FileInfo, - LgTemplate, - ILUFeaturesConfig, - DialogInfo, - DialogSchemaFile, - QnAFile, - SkillManifestFile, - RecognizerFile, - FormDialogSchema, -} from '@bfc/shared'; +import { LuIntentSection, LgFile, LuFile, QnASection, FileInfo, LgTemplate, ILUFeaturesConfig } from '@bfc/shared'; import { FileAsset } from '../persistence/types'; diff --git a/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts index 34589e680a..610da18579 100644 --- a/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts +++ b/Composer/packages/client/src/recoilModel/parsers/workers/calculator.worker.ts @@ -17,7 +17,7 @@ type MessageEvent = DifferenceMessage; const ctx: Worker = self as any; -function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) { +export function getDifferenceItems(target: FileAsset[], origin: FileAsset[]) { const changes1 = differenceWith(target, origin, isEqual); const changes2 = differenceWith(origin, target, isEqual); const deleted = changes2.filter((change) => !target.some((file) => change.id === file.id)); diff --git a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts index 253f293562..29a0807c6b 100644 --- a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts +++ b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts @@ -35,13 +35,18 @@ class FilePersistence { public async notify(currentAssets: BotAssets, previousAssets: BotAssets) { const fileChanges: IFileChange[] = await this.getAssetsChanges(currentAssets, previousAssets); + this.createTaskQueue(fileChanges); + + await this.flush(); + } + + public createTaskQueue(fileChanges: IFileChange[]) { for (const change of fileChanges) { if (!this._taskQueue[change.id]) { this._taskQueue[change.id] = []; } this._taskQueue[change.id].push(change); } - await this.flush(); } public async flush(): Promise { @@ -170,7 +175,7 @@ class FilePersistence { return []; } - private async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise { + public async getAssetsChanges(currentAssets: BotAssets, previousAssets: BotAssets): Promise { const files: [FileAsset[], FileAsset[], FileExtensions][] = [ [currentAssets.dialogs, previousAssets.dialogs, FileExtensions.Dialog], [currentAssets.dialogSchemas, previousAssets.dialogSchemas, FileExtensions.DialogSchema], diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index 854b2a7591..cd6e2f35dd 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -17,6 +17,12 @@ jest.mock('axios', () => { }; }); +jest.mock('../../parsers/calculator', () => { + return { + difference: require('../../parsers/workers/calculator.worker').getDifferenceItems, + }; +}); + describe('test persistence layer', () => { let filePersistence: FilePersistence; beforeEach(() => { @@ -40,8 +46,8 @@ describe('test persistence layer', () => { luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[], } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('new'); expect(JSON.parse(filePersistence.taskQueue['a.dialog.schema'][0].change).a).toBe('new schema'); expect(filePersistence.taskQueue['a.en-us.lg'][0].change).toBe('a.lg'); @@ -77,8 +83,8 @@ describe('test persistence layer', () => { ] as LuFile[], } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b'); expect(JSON.parse(filePersistence.taskQueue['b.dialog.schema'][0].change).b).toBe('b'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.lg'); @@ -113,8 +119,8 @@ describe('test persistence layer', () => { lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[], } as BotAssets; - filePersistence.notify(current, previous); - filePersistence.notify(current, previous); + const result = await filePersistence.getAssetsChanges(current, previous); + filePersistence.createTaskQueue(result); expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b.pre'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.pre.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.pre.lu'); From c06b4684244b97f5f216e3f3bbca6abd2c7741ad Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 20 Jan 2021 17:22:41 +0800 Subject: [PATCH 3/6] fix tests --- .../src/recoilModel/persistence/__test__/FilePersistence.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index cd6e2f35dd..b21a91e268 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -89,6 +89,7 @@ describe('test persistence layer', () => { expect(JSON.parse(filePersistence.taskQueue['b.dialog.schema'][0].change).b).toBe('b'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.lu'); + filePersistence.flush(); }); it('test notify remove', async () => { From 962433353ffd5278f63978b4d8061629a0ebe28e Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 20 Jan 2021 17:35:17 +0800 Subject: [PATCH 4/6] fix tests --- .../recoilModel/persistence/__test__/FilePersistence.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index b21a91e268..748b81d398 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -89,7 +89,8 @@ describe('test persistence layer', () => { expect(JSON.parse(filePersistence.taskQueue['b.dialog.schema'][0].change).b).toBe('b'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.lu'); - filePersistence.flush(); + await filePersistence.flush(); + expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0); }); it('test notify remove', async () => { From 5b41f532e00d0376659cb8d68a41860f5f9a4e5a Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 20 Jan 2021 21:46:32 +0800 Subject: [PATCH 5/6] add test coverage --- .../__test__/FilePersistence.test.ts | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index 748b81d398..a4ce889346 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -32,26 +32,42 @@ describe('test persistence layer', () => { it('test notify update', async () => { const previous = { projectId: 'test', - dialogs: ([{ id: 'a', content: { a: 'old' } }] as unknown) as DialogInfo[], + dialogs: [] as DialogInfo[], dialogSchemas: [{ id: 'a', content: { a: 'old schema' } }] as DialogSchemaFile[], lgFiles: [{ id: 'a.en-us', content: '' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: '' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: false } } as any, } as BotAssets; const current = { projectId: 'test', - dialogs: ([{ id: 'a', content: { a: 'new' } }] as unknown) as DialogInfo[], + dialogs: ([{ id: 'a', content: { a: 'create' } }] as unknown) as DialogInfo[], dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[], lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[], luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any, + } as BotAssets; + + const last = { + projectId: 'test', + dialogs: ([{ id: 'a', content: { a: 'update' } }] as unknown) as DialogInfo[], + dialogSchemas: [{ id: 'a', content: { a: 'new schema' } }] as DialogSchemaFile[], + lgFiles: [{ id: 'a.en-us', content: 'a.lg' }] as LgFile[], + luFiles: [{ id: 'a.en-us', content: 'a.lu' }] as LuFile[], + crossTrainConfig: { 'cross-train.config.json': { rootBot: true } } as any, } as BotAssets; const result = await filePersistence.getAssetsChanges(current, previous); filePersistence.createTaskQueue(result); - expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('new'); + expect(JSON.parse(filePersistence.taskQueue['a.dialog'][0].change).a).toBe('create'); expect(JSON.parse(filePersistence.taskQueue['a.dialog.schema'][0].change).a).toBe('new schema'); expect(filePersistence.taskQueue['a.en-us.lg'][0].change).toBe('a.lg'); expect(filePersistence.taskQueue['a.en-us.lu'][0].change).toBe('a.lu'); + const result1 = await filePersistence.getAssetsChanges(last, current); + filePersistence.createTaskQueue(result1); + filePersistence.flush(); + filePersistence.flush(); + expect(filePersistence.taskQueue['a.en-us.lu'].length).toBe(0); }); it('test notify create', async () => { @@ -126,5 +142,7 @@ describe('test persistence layer', () => { expect(JSON.parse(filePersistence.taskQueue['b.dialog'][0].change).b).toBe('b.pre'); expect(filePersistence.taskQueue['b.en-us.lg'][0].change).toBe('b.pre.lg'); expect(filePersistence.taskQueue['b.en-us.lu'][0].change).toBe('b.pre.lu'); + await filePersistence.flush(); + expect(filePersistence.taskQueue['b.en-us.lu'].length).toBe(0); }); }); From 00351b58cbebcaad35c85767a086b6dcc08798da Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 25 Jan 2021 10:58:20 +0800 Subject: [PATCH 6/6] update naming --- .../parsers/{calculator.ts => fileDiffCalculator.ts} | 4 ++-- .../client/src/recoilModel/persistence/FilePersistence.ts | 7 ++++--- .../persistence/__test__/FilePersistence.test.ts | 2 +- .../packages/client/src/recoilModel/persistence/types.ts | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) rename Composer/packages/client/src/recoilModel/parsers/{calculator.ts => fileDiffCalculator.ts} (80%) diff --git a/Composer/packages/client/src/recoilModel/parsers/calculator.ts b/Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts similarity index 80% rename from Composer/packages/client/src/recoilModel/parsers/calculator.ts rename to Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts index 04cc6417ba..e778cfbc7a 100644 --- a/Composer/packages/client/src/recoilModel/parsers/calculator.ts +++ b/Composer/packages/client/src/recoilModel/parsers/fileDiffCalculator.ts @@ -8,11 +8,11 @@ import { BaseWorker } from './baseWorker'; import { FilesDifferencePayload, CalculatorType } from './types'; // Wrapper class -class Calculator extends BaseWorker { +class FileDiffCalculator extends BaseWorker { difference(target: FileAsset[], origin: FileAsset[]) { const payload = { target, origin }; return this.sendMsg('difference', payload); } } -export default new Calculator(new Worker()); +export default new FileDiffCalculator(new Worker()); diff --git a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts index 29a0807c6b..90ec5d7b6d 100644 --- a/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts +++ b/Composer/packages/client/src/recoilModel/persistence/FilePersistence.ts @@ -4,9 +4,10 @@ import isEqual from 'lodash/isEqual'; import { DialogSetting, BotAssets, BotProjectFile, CrosstrainConfig } from '@bfc/shared'; import keys from 'lodash/keys'; +import fileDiffCalculator from '../parsers/fileDiffCalculator'; + import * as client from './http'; -import calculator from './../parsers/calculator'; -import { ChangeType, Difference, FileExtensions, IFileChange, FileAsset } from './types'; +import { ChangeType, FileDifference, FileExtensions, IFileChange, FileAsset } from './types'; class FilePersistence { private _taskQueue: { [id: string]: IFileChange[] } = {}; @@ -127,7 +128,7 @@ class FilePersistence { } private async getFilesChanges(current: FileAsset[], previous: FileAsset[], fileExtension: FileExtensions) { - const changes = (await calculator.difference(current, previous)) as Difference; + const changes = (await fileDiffCalculator.difference(current, previous)) as FileDifference; const updated = changes.updated.map((file) => this.createChange(file, fileExtension, ChangeType.UPDATE)); const added = changes.added.map((file) => this.createChange(file, fileExtension, ChangeType.CREATE)); const deleted = changes.deleted.map((file) => this.createChange(file, fileExtension, ChangeType.DELETE)); diff --git a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts index a4ce889346..7622939471 100644 --- a/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts +++ b/Composer/packages/client/src/recoilModel/persistence/__test__/FilePersistence.test.ts @@ -17,7 +17,7 @@ jest.mock('axios', () => { }; }); -jest.mock('../../parsers/calculator', () => { +jest.mock('../../parsers/fileDiffCalculator', () => { return { difference: require('../../parsers/workers/calculator.worker').getDifferenceItems, }; diff --git a/Composer/packages/client/src/recoilModel/persistence/types.ts b/Composer/packages/client/src/recoilModel/persistence/types.ts index 6c6f3bdeb8..801b1b0d25 100644 --- a/Composer/packages/client/src/recoilModel/persistence/types.ts +++ b/Composer/packages/client/src/recoilModel/persistence/types.ts @@ -52,7 +52,7 @@ export type FileAsset = | RecognizerFile | FormDialogSchema; -export type Difference = { +export type FileDifference = { updated: FileAsset[]; added: FileAsset[]; deleted: FileAsset[];