From 343eecfca8cf4ecfdab1c0c65cafa885b7d97eb5 Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 24 Feb 2020 22:52:30 +0800 Subject: [PATCH 01/14] Add cross train before luis publish --- Composer/packages/client/package.json | 2 +- Composer/packages/lib/indexers/package.json | 2 +- .../server/src/models/bot/botProject.ts | 2 +- .../server/src/models/bot/luPublisher.ts | 58 ++++++++++++++----- .../server/src/models/storage/interface.ts | 2 +- .../src/models/storage/localDiskStorage.ts | 2 +- Composer/packages/server/src/types/bf-lu.d.ts | 9 +++ .../language-understanding/package.json | 2 +- Composer/yarn.lock | 12 ++-- 9 files changed, 64 insertions(+), 27 deletions(-) create mode 100644 Composer/packages/server/src/types/bf-lu.d.ts diff --git a/Composer/packages/client/package.json b/Composer/packages/client/package.json index c0fcfb6d77..1b85957cea 100644 --- a/Composer/packages/client/package.json +++ b/Composer/packages/client/package.json @@ -20,7 +20,7 @@ "@bfc/extensions": "*", "@bfc/indexers": "*", "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.1.8", + "@bfcomposer/bf-lu": "^1.2.6", "@emotion/core": "^10.0.7", "@reach/router": "^1.2.1", "axios": "^0.18.0", diff --git a/Composer/packages/lib/indexers/package.json b/Composer/packages/lib/indexers/package.json index a8511930bc..275e8c3a74 100644 --- a/Composer/packages/lib/indexers/package.json +++ b/Composer/packages/lib/indexers/package.json @@ -29,7 +29,7 @@ }, "dependencies": { "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.1.8", + "@bfcomposer/bf-lu": "^1.2.6", "botbuilder-lg": "^4.8.0-preview.97252", "botframework-expressions": "4.7.0-preview.93464", "lodash": "^4.17.15" diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index 22e86e5579..b4ab817afa 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -522,7 +522,7 @@ export class BotProject { // load only from the data dir, otherwise may get "build" versions from // deployment process const root = this.dataDir; - const paths = await this.fileStorage.glob(pattern, root); + const paths = await this.fileStorage.glob([pattern, '!(generated/**)'], root); for (const filePath of paths.sort()) { const realFilePath: string = Path.join(root, filePath); diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index f117227f8a..f4c5379df0 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -3,13 +3,16 @@ import isEqual from 'lodash/isEqual'; import { runBuild } from '@bfcomposer/lubuild'; +import { crossTrain } from '@bfcomposer/bf-lu/lib/parser/lubuild'; import { LuFile } from '@bfc/indexers'; import { Path } from './../../utility/path'; import { IFileStorage } from './../storage/interface'; import { ILuisConfig, LuisStatus, FileUpdateType } from './interface'; const GENERATEDFOLDER = 'ComposerDialogs/generated'; +const INTERUPTION = 'ComposerDialogs/generated/interuption'; const LU_STATUS_FILE = 'luis.status.json'; +const CROSS_TRAIN_CONFIG = 'mapping_rules.json'; const DEFAULT_STATUS = { lastUpdateTime: 1, lastPublishTime: 0, // means unpublished @@ -17,6 +20,7 @@ const DEFAULT_STATUS = { export class LuPublisher { public botDir: string; public generatedFolderPath: string; + public interuptionFolderPath: string; public statusFile: string; public storage: IFileStorage; public config: ILuisConfig | null = null; @@ -27,6 +31,7 @@ export class LuPublisher { constructor(path: string, storage: IFileStorage) { this.botDir = path; this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER); + this.interuptionFolderPath = Path.join(this.botDir, INTERUPTION); this.statusFile = Path.join(this.generatedFolderPath, LU_STATUS_FILE); this.storage = storage; } @@ -80,24 +85,28 @@ export class LuPublisher { }; public publish = async (luFiles: LuFile[]) => { - const config = this._getConfig(luFiles); - if (config.models.length === 0) { - throw new Error('No luis file exist'); - } - const curTime = Date.now(); try { + await this._crossTrain(); + + const config = await this._getConfig(); + if (config.models.length === 0) { + throw new Error('No luis file exist'); + } + await runBuild(config); // update pubish status after sucessfully published + const curTime = Date.now(); luFiles.forEach(f => { this.status[f.relativePath].lastPublishTime = curTime; }); + await this.saveStatus(); + + await this._copyDialogsToTargetFolder(config); } catch (error) { - throw new Error(error?.body?.error?.message ?? 'Error publishing to LUIS.'); + throw new Error(error?.message ?? 'Error publishing to LUIS.'); } - - await this._copyDialogsToTargetFolder(config); }; public getUnpublisedFiles = (files: LuFile[]) => { @@ -132,6 +141,31 @@ export class LuPublisher { this.config.authoringKey = key; } }; + + public createCrossTrainConfig = async () => { + // ToDo: create real tree for cross train. Now add this data to test the bf-lu + const test = { + '../Main/Main.lu': { + rootDialog: true, + triggers: { + dia1: '../dia1/dia1.lu', + dia2: '../dia2/dia2.lu', + }, + }, + }; + await this.storage.writeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, JSON.stringify(test)); + }; + + private async _crossTrain() { + await this.createCrossTrainConfig(); + const result = await crossTrain.train( + this.botDir, + '_Interuption', + `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}` + ); + await crossTrain.writeFiles(result.luResult, this.interuptionFolderPath); + } + //delete files in generated folder private async _deleteGenerated(path: string) { if (await this.storage.exists(path)) { @@ -163,17 +197,15 @@ export class LuPublisher { }); }; - private _getConfig = (luFiles: LuFile[]) => { + private _getConfig = async () => { const luConfig: any = { ...this.config }; luConfig.models = []; luConfig.autodelete = true; luConfig.dialogs = true; luConfig.force = false; luConfig.folder = this.generatedFolderPath; - luFiles.forEach(file => { - luConfig.models.push(Path.resolve(this.botDir, file.relativePath)); - }); - + const paths = await this.storage.glob('**/*.lu', this.interuptionFolderPath); + luConfig.models = paths.map(filePath => Path.join(this.interuptionFolderPath, filePath)); return luConfig; }; } diff --git a/Composer/packages/server/src/models/storage/interface.ts b/Composer/packages/server/src/models/storage/interface.ts index bfefa3b4cc..6c5077b589 100644 --- a/Composer/packages/server/src/models/storage/interface.ts +++ b/Composer/packages/server/src/models/storage/interface.ts @@ -28,7 +28,7 @@ export interface IFileStorage { removeFile(path: string): Promise; mkDir(path: string, options?: MakeDirectoryOptions): Promise; rmDir(path: string): Promise; - glob(pattern: string, path: string): Promise; + glob(pattern: string | string[], path: string): Promise; copyFile(src: string, dest: string): Promise; rename(oldPath: string, newPath: string): Promise; } diff --git a/Composer/packages/server/src/models/storage/localDiskStorage.ts b/Composer/packages/server/src/models/storage/localDiskStorage.ts index 72bc116d0e..9213486128 100644 --- a/Composer/packages/server/src/models/storage/localDiskStorage.ts +++ b/Composer/packages/server/src/models/storage/localDiskStorage.ts @@ -63,7 +63,7 @@ export class LocalDiskStorage implements IFileStorage { await rmDir(path); } - async glob(pattern: string, path: string): Promise { + async glob(pattern: string | string[], path: string): Promise { return await glob(pattern, { cwd: path, dot: true }); } diff --git a/Composer/packages/server/src/types/bf-lu.d.ts b/Composer/packages/server/src/types/bf-lu.d.ts new file mode 100644 index 0000000000..cd8aa7ce49 --- /dev/null +++ b/Composer/packages/server/src/types/bf-lu.d.ts @@ -0,0 +1,9 @@ +/// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +declare module '@bfcomposer/bf-lu/lib/parser/lubuild' { + namespace crossTrain { + function train(input: any, intentName: any, configPath: any): any; + function writeFiles(fileIdToLuResourceMap: any, out: any): any; + } +} diff --git a/Composer/packages/tools/language-servers/language-understanding/package.json b/Composer/packages/tools/language-servers/language-understanding/package.json index fa8ebaddee..402caf28f9 100644 --- a/Composer/packages/tools/language-servers/language-understanding/package.json +++ b/Composer/packages/tools/language-servers/language-understanding/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "@microsoft/bf-cli-command": "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz", - "@bfcomposer/bf-lu": "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.1.8.tgz", + "@bfcomposer/bf-lu": "^1.2.6", "@types/node": "^12.0.4", "express": "^4.15.2", "monaco-editor-core": "^0.17.0", diff --git a/Composer/yarn.lock b/Composer/yarn.lock index 18aa59cc63..344c952967 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -2142,17 +2142,13 @@ tslib "^1.10.0" uuid "~3.3.3" -"@bfcomposer/bf-lu@^1.1.8", "@bfcomposer/bf-lu@https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.1.8.tgz": - version "1.1.8" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.1.8.tgz#a3c7767de038025157bdc8cf9f56b393f1428fd4" - integrity sha1-o8d2feA4AlFXvcjPn1azk/FCj9Q= +"@bfcomposer/bf-lu@^1.2.6": + version "1.2.6" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.6.tgz#de8346d6f1ce949abb5627fa8421f7f2d3e69b69" + integrity sha1-3oNG1vHOlJq7Vif6hCH38tPmm2k= dependencies: "@azure/cognitiveservices-luis-authoring" "3.0.1" "@azure/ms-rest-azure-js" "2.0.1" - "@microsoft/bf-cli-command" "1.0.0" - "@oclif/command" "~1.5.19" - "@oclif/config" "~1.13.3" - "@oclif/errors" "~1.2.2" antlr4 "^4.7.2" chalk "2.4.1" console-stream "^0.1.1" From 6d4a5002b4b7f9f33c17408f2cac02984cacad98 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 25 Feb 2020 16:40:41 +0800 Subject: [PATCH 02/14] update the publish flow --- .../server/src/models/bot/luPublisher.ts | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index f4c5379df0..3a788bbc9f 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -9,6 +9,7 @@ import { LuFile } from '@bfc/indexers'; import { Path } from './../../utility/path'; import { IFileStorage } from './../storage/interface'; import { ILuisConfig, LuisStatus, FileUpdateType } from './interface'; + const GENERATEDFOLDER = 'ComposerDialogs/generated'; const INTERUPTION = 'ComposerDialogs/generated/interuption'; const LU_STATUS_FILE = 'luis.status.json'; @@ -17,6 +18,7 @@ const DEFAULT_STATUS = { lastUpdateTime: 1, lastPublishTime: 0, // means unpublished }; + export class LuPublisher { public botDir: string; public generatedFolderPath: string; @@ -86,15 +88,21 @@ export class LuPublisher { public publish = async (luFiles: LuFile[]) => { try { + await this._createGeneratedDir(); + + //do cross train before publish await this._crossTrain(); - const config = await this._getConfig(); + const config = await this._getConfig(luFiles); if (config.models.length === 0) { throw new Error('No luis file exist'); } await runBuild(config); + //remove the cross train result + await this._cleanCrossTrain(); + // update pubish status after sucessfully published const curTime = Date.now(); luFiles.forEach(f => { @@ -102,8 +110,6 @@ export class LuPublisher { }); await this.saveStatus(); - - await this._copyDialogsToTargetFolder(config); } catch (error) { throw new Error(error?.message ?? 'Error publishing to LUIS.'); } @@ -131,7 +137,7 @@ export class LuPublisher { public setLuisConfig = async (config: ILuisConfig) => { if (!isEqual(config, this.config)) { this.config = config; - await this._deleteGenerated(this.generatedFolderPath); + await this._deleteDir(this.generatedFolderPath); this.resetStatus(); } }; @@ -156,6 +162,12 @@ export class LuPublisher { await this.storage.writeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, JSON.stringify(test)); }; + private async _createGeneratedDir() { + if (!(await this.storage.exists(this.generatedFolderPath))) { + await this.storage.mkDir(this.generatedFolderPath); + } + } + private async _crossTrain() { await this.createCrossTrainConfig(); const result = await crossTrain.train( @@ -167,45 +179,44 @@ export class LuPublisher { } //delete files in generated folder - private async _deleteGenerated(path: string) { + private async _deleteDir(path: string) { if (await this.storage.exists(path)) { const files = await this.storage.readDir(path); for (const file of files) { const curPath = Path.join(path, file); if ((await this.storage.stat(curPath)).isDir) { - await this._deleteGenerated(curPath); + await this._deleteDir(curPath); } else { await this.storage.removeFile(curPath); } } + await this.storage.rmDir(path); } } - private _copyDialogsToTargetFolder = async (config: any) => { - const defaultLanguage = config.defaultLanguage; - await config.models.forEach(async (filePath: string) => { - const baseName = Path.basename(filePath, '.lu'); - const rootPath = Path.dirname(filePath); - const currentPath = `${filePath}.dialog`; - const targetPath = Path.join(this.generatedFolderPath, `${baseName}.lu.dialog`); - const currentVariantPath = Path.join(rootPath, `${baseName}.${defaultLanguage}.lu.dialog`); - const targetVariantPath = Path.join(this.generatedFolderPath, `${baseName}.${defaultLanguage}.lu.dialog`); - await this.storage.copyFile(currentPath, targetPath); - await this.storage.copyFile(currentVariantPath, targetVariantPath); - await this.storage.removeFile(currentPath); - await this.storage.removeFile(currentVariantPath); - }); - }; - - private _getConfig = async () => { + private _getConfig = async (luFiles: LuFile[]) => { const luConfig: any = { ...this.config }; luConfig.models = []; luConfig.autodelete = true; luConfig.dialogs = true; luConfig.force = false; luConfig.folder = this.generatedFolderPath; + //add all lu file after cross train const paths = await this.storage.glob('**/*.lu', this.interuptionFolderPath); luConfig.models = paths.map(filePath => Path.join(this.interuptionFolderPath, filePath)); + + //add the lu file that are not in crossTrain config. + luFiles.forEach(file => { + if (~paths.indexOf(`${file.id}.lu`)) { + luConfig.models.push(Path.resolve(this.botDir, file.relativePath)); + } + }); + return luConfig; }; + + private async _cleanCrossTrain() { + await this.storage.removeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`); + await this._deleteDir(this.interuptionFolderPath); + } } From 66df2d61d3e8c28138564e1be589afc1e11c9034 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 25 Feb 2020 22:34:48 +0800 Subject: [PATCH 03/14] create cross train config --- .../lib/indexers/src/dialogIndexer.ts | 9 ++-- .../src/dialogUtils/extractIntentTriggers.ts | 27 +++++++++++ .../lib/indexers/src/dialogUtils/types.ts | 5 ++ Composer/packages/lib/indexers/src/type.ts | 2 + .../server/src/models/bot/botProject.ts | 1 + .../server/src/models/bot/luPublisher.ts | 47 ++++++++++++++----- 6 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts diff --git a/Composer/packages/lib/indexers/src/dialogIndexer.ts b/Composer/packages/lib/indexers/src/dialogIndexer.ts index 0940379b96..f35152a43c 100644 --- a/Composer/packages/lib/indexers/src/dialogIndexer.ts +++ b/Composer/packages/lib/indexers/src/dialogIndexer.ts @@ -11,7 +11,7 @@ import { JsonWalk, VisitorFunc } from './utils/jsonWalk'; import { getBaseName } from './utils/help'; import { Diagnostic } from './diagnostic'; import ExtractMemoryPaths from './dialogUtils/extractMemoryPaths'; - +import ExtractIntentTriggers from './dialogUtils/extractIntentTriggers'; // find out all lg templates given dialog function ExtractLgTemplates(id, dialog): LgTemplateJsonPath[] { const templates: LgTemplateJsonPath[] = []; @@ -91,7 +91,7 @@ function ExtractLuIntents(dialog): string[] { // find out all triggers given dialog function ExtractTriggers(dialog): ITrigger[] { - const trigers: ITrigger[] = []; + const triggers: ITrigger[] = []; /** * * @param path , jsonPath string * @param value , current node value * @@ -114,7 +114,7 @@ function ExtractTriggers(dialog): ITrigger[] { } else if (trigger.isIntent && has(rule, 'intent')) { trigger.displayName = rule.intent; } - trigers.push(trigger); + triggers.push(trigger); } }); return true; @@ -122,7 +122,7 @@ function ExtractTriggers(dialog): ITrigger[] { return false; }; JsonWalk('$', dialog, visitor); - return trigers; + return triggers; } // find out all referred dialog @@ -200,6 +200,7 @@ function parse(id: string, content: any, schema: any) { luFile: getBaseName(luFile, '.lu'), lgFile: getBaseName(lgFile, '.lg'), triggers: ExtractTriggers(content), + intentTriggers: ExtractIntentTriggers(content), }; } diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts new file mode 100644 index 0000000000..feb45492c8 --- /dev/null +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import { SDKTypes } from '@bfc/shared'; + +import { VisitorFunc, JsonWalk } from '../utils/jsonWalk'; + +import { IIntentTrigger } from './types'; + +// find out all properties from given dialog +function ExtractIntentTriggers(value: any): IIntentTrigger[] { + const triggers: IIntentTrigger[] = []; + + const visitor: VisitorFunc = (path: string, value: any): boolean => { + if (value.$type === SDKTypes.OnIntent && value.actions[0]?.$type === SDKTypes.BeginDialog) { + triggers.push({ + intent: value.intent, + dialog: value.actions[0].dialog, + }); + } + return true; + }; + JsonWalk('$', value, visitor); + + return triggers; +} + +export default ExtractIntentTriggers; diff --git a/Composer/packages/lib/indexers/src/dialogUtils/types.ts b/Composer/packages/lib/indexers/src/dialogUtils/types.ts index 4e7960f655..0eb1b88f4b 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/types.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/types.ts @@ -23,4 +23,9 @@ export interface IExpressionProperties { }; } +export interface IIntentTrigger { + intent: string; + dialog: string; +} + export type CheckerFunc = (path: string, value: any, type: string, schema: any) => Diagnostic[] | null; // error msg diff --git a/Composer/packages/lib/indexers/src/type.ts b/Composer/packages/lib/indexers/src/type.ts index c02d2d5498..96de27898f 100644 --- a/Composer/packages/lib/indexers/src/type.ts +++ b/Composer/packages/lib/indexers/src/type.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { Diagnostic } from './diagnostic'; +import { IIntentTrigger } from './dialogUtils/types'; export interface FileInfo { name: string; @@ -31,6 +32,7 @@ export interface DialogInfo { relativePath: string; userDefinedVariables: string[]; triggers: ITrigger[]; + intentTriggers: IIntentTrigger[]; } export interface LgTemplateJsonPath { diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index b4ab817afa..3f74ce1996 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -353,6 +353,7 @@ export class BotProject { } if (unpublished.length > 0) { + await this.luPublisher.setCrossTrainConfig(this.dialogs); await this.luPublisher.publish(unpublished); } diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index 3a788bbc9f..5a332a34fd 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -4,7 +4,7 @@ import isEqual from 'lodash/isEqual'; import { runBuild } from '@bfcomposer/lubuild'; import { crossTrain } from '@bfcomposer/bf-lu/lib/parser/lubuild'; -import { LuFile } from '@bfc/indexers'; +import { LuFile, DialogInfo } from '@bfc/indexers'; import { Path } from './../../utility/path'; import { IFileStorage } from './../storage/interface'; @@ -26,6 +26,7 @@ export class LuPublisher { public statusFile: string; public storage: IFileStorage; public config: ILuisConfig | null = null; + public crossTrainMapRule: { [key: string]: string } = {}; // key: filePath relative to bot dir // value: lastUpdateTime && lastPublishTime @@ -90,6 +91,8 @@ export class LuPublisher { try { await this._createGeneratedDir(); + await this.createCrossTrainConfig(); + //do cross train before publish await this._crossTrain(); @@ -148,20 +151,38 @@ export class LuPublisher { } }; - public createCrossTrainConfig = async () => { + public setCrossTrainConfig = async (dialogs: DialogInfo[]) => { // ToDo: create real tree for cross train. Now add this data to test the bf-lu - const test = { - '../Main/Main.lu': { - rootDialog: true, - triggers: { - dia1: '../dia1/dia1.lu', - dia2: '../dia2/dia2.lu', - }, - }, - }; - await this.storage.writeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, JSON.stringify(test)); + const rootDialog = dialogs.find(dialog => dialog.isRoot); + if (rootDialog?.intentTriggers.length) { + this.crossTrainMapRule = this._createTree(rootDialog, dialogs); + } }; + //write config to generated folder + public async createCrossTrainConfig() { + await this.storage.writeFile( + `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, + JSON.stringify(this.crossTrainMapRule) + ); + } + + private _createTree(dialog: DialogInfo, dialogs: DialogInfo[]) { + let result = {}; + const key = dialog.relativePath.replace('.dialog', '.lu'); + dialog.intentTriggers.forEach(temp => { + const target = dialogs.find(dialog => dialog.id === temp.dialog); + if (target && target.content?.recognizer) { + if (!result[key].triggers) result[key].triggers = {}; + result[key].triggers[temp.intent] = target.relativePath.replace('.dialog', '.lu'); + result = { ...result, ...this._createTree(target, dialogs) }; + } + }); + + if (dialog.isRoot && result[key]) result[key].rootDialog = true; + return result; + } + private async _createGeneratedDir() { if (!(await this.storage.exists(this.generatedFolderPath))) { await this.storage.mkDir(this.generatedFolderPath); @@ -169,7 +190,7 @@ export class LuPublisher { } private async _crossTrain() { - await this.createCrossTrainConfig(); + //await this.createCrossTrainConfig(); const result = await crossTrain.train( this.botDir, '_Interuption', From 80c16c45cf33a5bf7ad8b2898a76e1f2f2c197e7 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 26 Feb 2020 12:01:44 +0800 Subject: [PATCH 04/14] fix some bug --- .../src/dialogUtils/extractIntentTriggers.ts | 3 ++- .../packages/server/src/models/bot/luPublisher.ts | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts index feb45492c8..8a56586425 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -16,8 +16,9 @@ function ExtractIntentTriggers(value: any): IIntentTrigger[] { intent: value.intent, dialog: value.actions[0].dialog, }); + return true; } - return true; + return false; }; JsonWalk('$', value, visitor); diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index 5a332a34fd..436d99b991 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -167,14 +167,21 @@ export class LuPublisher { ); } + private _createPath(dialogPath: string) { + const path = dialogPath.replace('.dialog', '.lu'); + const absolutePath = Path.join(this.botDir, path); + const relativePath = Path.relative(this.generatedFolderPath, absolutePath); + return relativePath; + } + private _createTree(dialog: DialogInfo, dialogs: DialogInfo[]) { let result = {}; - const key = dialog.relativePath.replace('.dialog', '.lu'); + const key = this._createPath(dialog.relativePath); dialog.intentTriggers.forEach(temp => { const target = dialogs.find(dialog => dialog.id === temp.dialog); if (target && target.content?.recognizer) { - if (!result[key].triggers) result[key].triggers = {}; - result[key].triggers[temp.intent] = target.relativePath.replace('.dialog', '.lu'); + if (!result[key]) result[key] = { triggers: {} }; + result[key].triggers[temp.intent] = this._createPath(target.relativePath); result = { ...result, ...this._createTree(target, dialogs) }; } }); From da9edb7b3fa3ae7c565b40f7b5c60c0512de05fe Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 26 Feb 2020 13:33:53 +0800 Subject: [PATCH 05/14] update the lubuild version --- Composer/packages/server/package.json | 2 +- Composer/yarn.lock | 42 +++++++-------------------- 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index 3920015655..a65676558d 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -57,7 +57,7 @@ "@bfc/lg-languageserver": "*", "@bfc/lu-languageserver": "*", "@bfc/shared": "*", - "@bfcomposer/lubuild": "1.1.2-preview", + "@bfcomposer/lubuild": "1.1.4-preview", "archiver": "^3.0.0", "axios": "^0.18.0", "azure-storage": "^2.10.3", diff --git a/Composer/yarn.lock b/Composer/yarn.lock index 344c952967..ec17a91faa 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -2120,29 +2120,7 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@bfcomposer/bf-lu@1.1.2": - version "1.1.2" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.1.2.tgz#0dcd1efa9f839daefe9e2dfc6f4b6db731b701a3" - integrity sha1-Dc0e+p+Dna7+ni38b0tttzG3AaM= - dependencies: - "@microsoft/bf-cli-command" "1.0.0" - "@oclif/command" "~1.5.19" - "@oclif/config" "~1.13.3" - "@oclif/errors" "~1.2.2" - antlr4 "^4.7.2" - chalk "2.4.1" - console-stream "^0.1.1" - deep-equal "^1.0.1" - fs-extra "^8.1.0" - get-stdin "^6.0.0" - intercept-stdout "^0.1.2" - lodash "^4.17.15" - node-fetch "^2.1.2" - semver "^5.5.1" - tslib "^1.10.0" - uuid "~3.3.3" - -"@bfcomposer/bf-lu@^1.2.6": +"@bfcomposer/bf-lu@1.2.6", "@bfcomposer/bf-lu@^1.2.6": version "1.2.6" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.6.tgz#de8346d6f1ce949abb5627fa8421f7f2d3e69b69" integrity sha1-3oNG1vHOlJq7Vif6hCH38tPmm2k= @@ -2162,13 +2140,13 @@ semver "^5.5.1" tslib "^1.10.0" -"@bfcomposer/lubuild@1.1.2-preview": - version "1.1.2-preview" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/lubuild/-/@bfcomposer/lubuild-1.1.2-preview.tgz#426e5e109c2d6752745dc363aa824a653a3757f7" - integrity sha1-Qm5eEJwtZ1J0XcNjqoJKZTo3V/c= +"@bfcomposer/lubuild@1.1.4-preview": + version "1.1.4-preview" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/lubuild/-/@bfcomposer/lubuild-1.1.4-preview.tgz#6e318bfa641a8f1cb5c3b588e5bc5782a1227378" + integrity sha1-bjGL+mQajxy1w7WI5bxXgqEic3g= dependencies: "@azure/ms-rest-js" "1.7.0" - "@bfcomposer/bf-lu" "1.1.2" + "@bfcomposer/bf-lu" "1.2.6" async-file "^2.0.2" await-delay "^1.0.0" chalk "^2.4.2" @@ -2661,7 +2639,7 @@ "@types/istanbul-reports" "^1.1.1" "@types/yargs" "^13.0.0" -"@microsoft/bf-cli-command@1.0.0", "@microsoft/bf-cli-command@https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz": +"@microsoft/bf-cli-command@https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz": version "1.0.0" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz#0d681e93690c4e872bf1181a3ff02f094830929e" integrity sha1-DWgek2kMTocr8RgaP/AvCUgwkp4= @@ -2698,7 +2676,7 @@ resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw== -"@oclif/command@^1.5.13", "@oclif/command@~1.5.13", "@oclif/command@~1.5.19": +"@oclif/command@^1.5.13", "@oclif/command@~1.5.13": version "1.5.19" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/command/-/@oclif/command-1.5.19.tgz#13f472450eb83bd6c6871a164c03eadb5e1a07ed" integrity sha1-E/RyRQ64O9bGhxoWTAPq214aB+0= @@ -2710,7 +2688,7 @@ debug "^4.1.1" semver "^5.6.0" -"@oclif/config@^1", "@oclif/config@~1.13.3": +"@oclif/config@^1": version "1.13.3" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/config/-/@oclif/config-1.13.3.tgz#1b13e18d0e4242ddbd9cbd100f0eec819aa2bf8c" integrity sha1-GxPhjQ5CQt29nL0QDw7sgZqiv4w= @@ -16962,7 +16940,7 @@ uuid@^3.0.0, uuid@^3.0.1, uuid@^3.2.1, uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -uuid@^3.3.3, uuid@~3.3.3: +uuid@^3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" integrity sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ== From bcc59e65e4c22ad6795f61366c36a2a6a2081cba Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 2 Mar 2020 16:52:51 +0800 Subject: [PATCH 06/14] update the bf-lu version and remove the composer's file update check --- Composer/packages/client/package.json | 2 +- Composer/packages/lib/indexers/package.json | 2 +- .../packages/lib/indexers/src/luIndexer.ts | 2 +- .../lib/indexers/src/types/bf-lu.d.ts | 2 +- .../packages/lib/indexers/src/utils/luUtil.ts | 2 +- .../server/src/models/bot/botProject.ts | 53 ++------- .../server/src/models/bot/interface.ts | 7 -- .../server/src/models/bot/luPublisher.ts | 105 +++--------------- .../language-understanding/package.json | 2 +- Composer/yarn.lock | 56 ++++++++-- 10 files changed, 77 insertions(+), 156 deletions(-) diff --git a/Composer/packages/client/package.json b/Composer/packages/client/package.json index 1b85957cea..58a570af7d 100644 --- a/Composer/packages/client/package.json +++ b/Composer/packages/client/package.json @@ -20,7 +20,7 @@ "@bfc/extensions": "*", "@bfc/indexers": "*", "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.2.6", + "@bfcomposer/bf-lu": "^1.2.7", "@emotion/core": "^10.0.7", "@reach/router": "^1.2.1", "axios": "^0.18.0", diff --git a/Composer/packages/lib/indexers/package.json b/Composer/packages/lib/indexers/package.json index 275e8c3a74..11c9dce41a 100644 --- a/Composer/packages/lib/indexers/package.json +++ b/Composer/packages/lib/indexers/package.json @@ -29,7 +29,7 @@ }, "dependencies": { "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.2.6", + "@bfcomposer/bf-lu": "^1.2.7", "botbuilder-lg": "^4.8.0-preview.97252", "botframework-expressions": "4.7.0-preview.93464", "lodash": "^4.17.15" diff --git a/Composer/packages/lib/indexers/src/luIndexer.ts b/Composer/packages/lib/indexers/src/luIndexer.ts index 49a579bcc0..94d1be0460 100644 --- a/Composer/packages/lib/indexers/src/luIndexer.ts +++ b/Composer/packages/lib/indexers/src/luIndexer.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser'; +import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser/composerindex'; import get from 'lodash/get'; import { FileInfo, LuFile, LuParsed, LuSectionTypes, LuIntentSection } from './type'; diff --git a/Composer/packages/lib/indexers/src/types/bf-lu.d.ts b/Composer/packages/lib/indexers/src/types/bf-lu.d.ts index 2383096793..f8df414a4c 100644 --- a/Composer/packages/lib/indexers/src/types/bf-lu.d.ts +++ b/Composer/packages/lib/indexers/src/types/bf-lu.d.ts @@ -1,7 +1,7 @@ /// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -declare module '@bfcomposer/bf-lu/lib/parser' { +declare module '@bfcomposer/bf-lu/lib/parser/composerindex' { namespace parser { function parseFile(fileContent: any, log: any, locale: any): any; function validateLUISBlob(LUISJSONBlob: any): any; diff --git a/Composer/packages/lib/indexers/src/utils/luUtil.ts b/Composer/packages/lib/indexers/src/utils/luUtil.ts index ef0867686d..4db52243af 100644 --- a/Composer/packages/lib/indexers/src/utils/luUtil.ts +++ b/Composer/packages/lib/indexers/src/utils/luUtil.ts @@ -7,7 +7,7 @@ * for more usage detail, please check client/__tests__/utils/luUtil.test.ts */ -import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser'; +import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser/composerindex'; import isEmpty from 'lodash/isEmpty'; import { LuIntentSection, LuSectionTypes } from '../type'; diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index 3f74ce1996..ca59180b2c 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -23,7 +23,7 @@ import { ISettingManager, OBFUSCATED_VALUE } from '../settings'; import log from '../../logger'; import { IFileStorage } from './../storage/interface'; -import { LocationRef, LuisStatus, FileUpdateType } from './interface'; +import { LocationRef } from './interface'; import { LuPublisher } from './luPublisher'; import { DialogSetting } from './interface'; @@ -89,7 +89,6 @@ export class BotProject { if (this.settings) { await this.luPublisher.setLuisConfig(this.settings.luis); } - await this.luPublisher.loadStatus(this.luFiles.map(f => f.relativePath)); }; public getIndexes = () => { @@ -99,7 +98,7 @@ export class BotProject { location: this.dir, dialogs: this.dialogs, lgFiles: this.lgFiles, - luFiles: this.mergeLuStatus(this.luFiles, this.luPublisher.status), + luFiles: this.luFiles, schemas: this.getSchemas(), botEnvironment: this.environment.getEnvironmentName(this.name), settings: this.settings, @@ -133,25 +132,6 @@ export class BotProject { await this.luPublisher.setLuisConfig(config.luis); }; - // merge the status managed by luPublisher to the LuFile structure to keep a - // unified interface - private mergeLuStatus = ( - luFiles: LuFile[], - luStatus: { - [key: string]: LuisStatus; - } - ) => { - return luFiles.map(x => { - if (!luStatus[x.relativePath]) { - throw new Error(`No luis status for lu file ${x.relativePath}`); - } - return { - ...x, - status: luStatus[x.relativePath], - }; - }); - }; - public getSchemas = () => { let editorSchema = this.defaultEditorSchema; let sdkSchema = this.defaultSDKSchema; @@ -305,9 +285,8 @@ export class BotProject { } await this._updateFile(luFile.relativePath, content); - await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.UPDATE); - return this.mergeLuStatus(this.luFiles, this.luPublisher.status); + return this.luFiles; }; public createLuFile = async (id: string, content: string, dir: string = this.defaultDir(id)): Promise => { @@ -319,8 +298,7 @@ export class BotProject { // TODO: validate before save await this._createFile(relativePath, content); - await this.luPublisher.onFileChange(relativePath, FileUpdateType.CREATE); // let publisher know that some files changed - return this.mergeLuStatus(this.luFiles, this.luPublisher.status); // return a merged LUFile always + return this.luFiles; // return a merged LUFile always }; public removeLuFile = async (id: string): Promise => { @@ -331,42 +309,31 @@ export class BotProject { await this._removeFile(luFile.relativePath); - await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.DELETE); await this._cleanUp(luFile.relativePath); - return this.mergeLuStatus(this.luFiles, this.luPublisher.status); + return this.luFiles; }; public publishLuis = async (authoringKey: string) => { this.luPublisher.setAuthoringKey(authoringKey); const referred = this.luFiles.filter(this.isReferred); - const unpublished = this.luPublisher.getUnpublisedFiles(referred); - const invalidLuFile = unpublished.filter(file => file.diagnostics.length !== 0); + const invalidLuFile = referred.filter(file => file.diagnostics.length !== 0); if (invalidLuFile.length !== 0) { const msg = this.generateErrorMessage(invalidLuFile); throw new Error(`The Following LuFile(s) are invalid: \n` + msg); } - const emptyLuFiles = unpublished.filter(this.isLuFileEmpty); + const emptyLuFiles = referred.filter(this.isLuFileEmpty); if (emptyLuFiles.length !== 0) { const msg = emptyLuFiles.map(file => file.id).join(' '); throw new Error(`You have the following empty LuFile(s): ` + msg); } - if (unpublished.length > 0) { + if (referred.length > 0) { await this.luPublisher.setCrossTrainConfig(this.dialogs); - await this.luPublisher.publish(unpublished); + await this.luPublisher.publish(referred); } - return this.mergeLuStatus(this.luFiles, this.luPublisher.status); - }; - - public checkLuisPublished = () => { - const referredLuFiles = this.luFiles.filter(this.isReferred); - if (referredLuFiles.length <= 0) { - return true; - } else { - return this.luPublisher.checkLuisPublised(referredLuFiles); - } + return this.luFiles; }; public cloneFiles = async (locationRef: LocationRef): Promise => { diff --git a/Composer/packages/server/src/models/bot/interface.ts b/Composer/packages/server/src/models/bot/interface.ts index 92c75875fd..4cfa754c16 100644 --- a/Composer/packages/server/src/models/bot/interface.ts +++ b/Composer/packages/server/src/models/bot/interface.ts @@ -14,11 +14,6 @@ export interface ILuisSettings { }; } -export interface LuisStatus { - lastUpdateTime: number; - lastPublishTime: number; -} - // we will probably also use this interface to consolidate the processing of lu\lg\dialog export enum FileUpdateType { CREATE = 'create', @@ -40,8 +35,6 @@ export interface IOperationLUFile { relativePath?: string; content?: string; intents: []; - lastUpdateTime?: number; - lastPublishTime?: number; [key: string]: any; } diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index 436d99b991..897f62ea2f 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -3,90 +3,37 @@ import isEqual from 'lodash/isEqual'; import { runBuild } from '@bfcomposer/lubuild'; -import { crossTrain } from '@bfcomposer/bf-lu/lib/parser/lubuild'; import { LuFile, DialogInfo } from '@bfc/indexers'; import { Path } from './../../utility/path'; import { IFileStorage } from './../storage/interface'; -import { ILuisConfig, LuisStatus, FileUpdateType } from './interface'; +import { ILuisConfig } from './interface'; -const GENERATEDFOLDER = 'ComposerDialogs/generated'; -const INTERUPTION = 'ComposerDialogs/generated/interuption'; -const LU_STATUS_FILE = 'luis.status.json'; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const crossTrain = require('@bfcomposer/bf-lu/lib/parser/cross-train/cross-train.js'); + +const DIALOGS_FOLDER = 'ComposerDialogs'; +const GENERATEDFOLDER = 'generated'; +const INTERUPTION = 'interuption'; const CROSS_TRAIN_CONFIG = 'mapping_rules.json'; -const DEFAULT_STATUS = { - lastUpdateTime: 1, - lastPublishTime: 0, // means unpublished -}; export class LuPublisher { public botDir: string; + public dialogsDir: string; public generatedFolderPath: string; public interuptionFolderPath: string; - public statusFile: string; public storage: IFileStorage; public config: ILuisConfig | null = null; public crossTrainMapRule: { [key: string]: string } = {}; - // key: filePath relative to bot dir - // value: lastUpdateTime && lastPublishTime - public status: { [key: string]: LuisStatus } = {}; constructor(path: string, storage: IFileStorage) { this.botDir = path; - this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER); - this.interuptionFolderPath = Path.join(this.botDir, INTERUPTION); - this.statusFile = Path.join(this.generatedFolderPath, LU_STATUS_FILE); + this.dialogsDir = Path.join(this.botDir, DIALOGS_FOLDER); + this.generatedFolderPath = Path.join(this.dialogsDir, GENERATEDFOLDER); + this.interuptionFolderPath = Path.join(this.generatedFolderPath, INTERUPTION); this.storage = storage; } - // load luis status from luis.status.json - public loadStatus = async (files: string[] = []) => { - if (await this.storage.exists(this.statusFile)) { - const content = await this.storage.readFile(this.statusFile); - this.status = JSON.parse(content); - } - - // make sure all LU file have an initial value - files.forEach(f => { - if (!this.status[f]) { - this.status[f] = { ...DEFAULT_STATUS }; // use ... ensure don't referred to the same object - } - }); - return this.status; - }; - - // reset status when config changed, because status don't represent the current config - public resetStatus = () => { - for (const key in this.status) { - this.status[key] = { ...DEFAULT_STATUS }; - } - }; - - public saveStatus = async () => { - if (!(await this.storage.exists(this.generatedFolderPath))) { - await this.storage.mkDir(this.generatedFolderPath); - } - await this.storage.writeFile(this.statusFile, JSON.stringify(this.status, null, 2)); - }; - - public onFileChange = async (relativePath: string, type: FileUpdateType) => { - switch (type) { - case FileUpdateType.CREATE: - this.status[relativePath] = { - lastUpdateTime: Date.now(), - lastPublishTime: 0, // unpublished - }; - break; - case FileUpdateType.UPDATE: - this.status[relativePath].lastUpdateTime = Date.now(); - break; - case FileUpdateType.DELETE: - delete this.status[relativePath]; - break; - } - await this.saveStatus(); - }; - public publish = async (luFiles: LuFile[]) => { try { await this._createGeneratedDir(); @@ -105,43 +52,17 @@ export class LuPublisher { //remove the cross train result await this._cleanCrossTrain(); - - // update pubish status after sucessfully published - const curTime = Date.now(); - luFiles.forEach(f => { - this.status[f.relativePath].lastPublishTime = curTime; - }); - - await this.saveStatus(); } catch (error) { - throw new Error(error?.message ?? 'Error publishing to LUIS.'); + throw new Error(error.message ?? error.text ?? 'Error publishing to LUIS.'); } }; - public getUnpublisedFiles = (files: LuFile[]) => { - // unpublished means either - // 1. there is no status tracking - // 2. the status shows that lastPublishTime < lastUpdateTime - return files.filter(f => { - return ( - !this.status[f.relativePath] || - this.status[f.relativePath].lastPublishTime <= this.status[f.relativePath].lastUpdateTime - ); - }); - }; - - public checkLuisPublised = (files: LuFile[]) => { - const unpublished = this.getUnpublisedFiles(files); - return unpublished.length === 0; - }; - public getLuisConfig = () => this.config; public setLuisConfig = async (config: ILuisConfig) => { if (!isEqual(config, this.config)) { this.config = config; await this._deleteDir(this.generatedFolderPath); - this.resetStatus(); } }; @@ -199,7 +120,7 @@ export class LuPublisher { private async _crossTrain() { //await this.createCrossTrainConfig(); const result = await crossTrain.train( - this.botDir, + this.dialogsDir, '_Interuption', `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}` ); diff --git a/Composer/packages/tools/language-servers/language-understanding/package.json b/Composer/packages/tools/language-servers/language-understanding/package.json index c03b97c959..c151e7a31f 100644 --- a/Composer/packages/tools/language-servers/language-understanding/package.json +++ b/Composer/packages/tools/language-servers/language-understanding/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "@microsoft/bf-cli-command": "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz", - "@bfcomposer/bf-lu": "^1.2.6", + "@bfcomposer/bf-lu": "^1.2.7", "@types/node": "^12.0.4", "express": "^4.15.2", "monaco-editor-core": "^0.17.0", diff --git a/Composer/yarn.lock b/Composer/yarn.lock index e59323f425..98697c3cf8 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -1972,6 +1972,13 @@ dependencies: regenerator-runtime "^0.13.2" +"@babel/runtime@^7.2.0", "@babel/runtime@^7.8.4": + version "7.8.4" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.8.4.tgz#d79f5a2040f7caa24d53e563aad49cbc05581308" + integrity sha512-neAp3zt80trRVBI1x0azq6c57aNBqYZH8KhMm3TaB7wEI5Q4A2SHfBHE8w9gOhI/lrqxtEbXZgQIrHP+wvSGwQ== + dependencies: + regenerator-runtime "^0.13.2" + "@babel/runtime@^7.3.1", "@babel/runtime@^7.3.4": version "7.4.2" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.4.2.tgz#f5ab6897320f16decd855eed70b705908a313fe8" @@ -2014,13 +2021,6 @@ dependencies: regenerator-runtime "^0.13.2" -"@babel/runtime@^7.8.4": - version "7.8.4" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.8.4.tgz#d79f5a2040f7caa24d53e563aad49cbc05581308" - integrity sha512-neAp3zt80trRVBI1x0azq6c57aNBqYZH8KhMm3TaB7wEI5Q4A2SHfBHE8w9gOhI/lrqxtEbXZgQIrHP+wvSGwQ== - dependencies: - regenerator-runtime "^0.13.2" - "@babel/template@^7.0.0", "@babel/template@^7.1.0", "@babel/template@^7.2.2", "@babel/template@^7.4.0": version "7.4.0" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.4.0.tgz#12474e9c077bae585c5d835a95c0b0b790c25c8b" @@ -2120,7 +2120,7 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@bfcomposer/bf-lu@1.2.6", "@bfcomposer/bf-lu@^1.2.6": +"@bfcomposer/bf-lu@1.2.6": version "1.2.6" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.6.tgz#de8346d6f1ce949abb5627fa8421f7f2d3e69b69" integrity sha1-3oNG1vHOlJq7Vif6hCH38tPmm2k= @@ -2140,6 +2140,26 @@ semver "^5.5.1" tslib "^1.10.0" +"@bfcomposer/bf-lu@^1.2.7": + version "1.2.7" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.7.tgz#7bdcec5660c894164cc17f7065e76981bd57bcea" + integrity sha1-e9zsVmDIlBZMwX9wZedpgb1XvOo= + dependencies: + "@azure/cognitiveservices-luis-authoring" "3.0.1" + "@azure/ms-rest-azure-js" "2.0.1" + antlr4 "^4.7.2" + chalk "2.4.1" + console-stream "^0.1.1" + deep-equal "^1.0.1" + delay "^4.3.0" + fs-extra "^8.1.0" + get-stdin "^6.0.0" + intercept-stdout "^0.1.2" + lodash "^4.17.15" + node-fetch "^2.1.2" + semver "^5.5.1" + tslib "^1.10.0" + "@bfcomposer/lubuild@1.1.4-preview": version "1.1.4-preview" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/lubuild/-/@bfcomposer/lubuild-1.1.4-preview.tgz#6e318bfa641a8f1cb5c3b588e5bc5782a1227378" @@ -8843,6 +8863,11 @@ get-caller-file@^2.0.1: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-node-dimensions@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/get-node-dimensions/-/get-node-dimensions-1.2.1.tgz#fb7b4bb57060fb4247dd51c9d690dfbec56b0823" + integrity sha512-2MSPMu7S1iOTL+BOa6K1S62hB2zUAYNF/lV0gSVlOaacd087lc6nR1H1r0e3B1CerTo+RceOmi1iJW+vp21xcQ== + get-own-enumerable-property-symbols@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz#b877b49a5c16aefac3655f2ed2ea5b684df8d203" @@ -14448,6 +14473,16 @@ react-lifecycles-compat@^3.0.4: resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== +react-measure@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/react-measure/-/react-measure-2.3.0.tgz#75835d39abec9ae13517f35a819c160997a7a44e" + integrity sha512-dwAvmiOeblj5Dvpnk8Jm7Q8B4THF/f1l1HtKVi0XDecsG6LXwGvzV5R1H32kq3TW6RW64OAf5aoQxpIgLa4z8A== + dependencies: + "@babel/runtime" "^7.2.0" + get-node-dimensions "^1.2.1" + prop-types "^15.6.2" + resize-observer-polyfill "^1.5.0" + react-testing-library@^6.0.1: version "6.0.2" resolved "https://registry.yarnpkg.com/react-testing-library/-/react-testing-library-6.0.2.tgz#afd7ddaa174e21cf672605e4e4f6f8156c4c9ef9" @@ -14900,6 +14935,11 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= +resize-observer-polyfill@^1.5.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464" + integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== + resolve-cwd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a" From 43ee93607d8ee14d984b57612787279166aafb2f Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 2 Mar 2020 17:24:25 +0800 Subject: [PATCH 07/14] fix bug: the file add error --- Composer/packages/server/src/models/bot/luPublisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index 897f62ea2f..bff3a64a80 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -156,7 +156,7 @@ export class LuPublisher { //add the lu file that are not in crossTrain config. luFiles.forEach(file => { - if (~paths.indexOf(`${file.id}.lu`)) { + if (paths.indexOf(`${file.id}.lu`) === -1) { luConfig.models.push(Path.resolve(this.botDir, file.relativePath)); } }); From fa03e12d4d7be7fc35dc8b49c536690210e1293b Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 2 Mar 2020 18:13:44 +0800 Subject: [PATCH 08/14] remove the status test --- .../models/bot/luisPublisher.test.ts | 68 ------------------- .../server/src/models/bot/luPublisher.ts | 2 +- 2 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 Composer/packages/server/__tests__/models/bot/luisPublisher.test.ts diff --git a/Composer/packages/server/__tests__/models/bot/luisPublisher.test.ts b/Composer/packages/server/__tests__/models/bot/luisPublisher.test.ts deleted file mode 100644 index 45c32fcefe..0000000000 --- a/Composer/packages/server/__tests__/models/bot/luisPublisher.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { FileUpdateType } from '../../../src/models/bot/interface'; -import { Path } from '../../../src/utility/path'; - -import { LuPublisher } from './../../../src/models/bot/luPublisher'; -import service from './../../../src/services/storage'; - -const botDir = Path.join(__dirname, '../../mocks/samplebots/bot1'); -const storage = service.getStorageClient('default'); - -describe('luis status management', () => { - it('will get luis status', async () => { - const luPublisher = new LuPublisher(botDir, storage); - const status = await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu', 'bot1/Main.lu']); - expect(status['bot1/a.lu'].lastUpdateTime).toBe(1); - expect(status['bot1/a.lu'].lastPublishTime).toBe(0); - }); - - it('can update luis status', async () => { - const luPublisher = new LuPublisher(botDir, storage); - - await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu', 'bot1/Main.lu']); - const oldUpdateTime = luPublisher.status['bot1/a.lu'].lastUpdateTime; - - await luPublisher.onFileChange('bot1/a.lu', FileUpdateType.UPDATE); - const newUpdateTime = luPublisher.status['bot1/a.lu'].lastUpdateTime; - // update should increase the update time - expect(newUpdateTime).toBeGreaterThan(oldUpdateTime); - }); -}); - -describe('get unpublishedFiles', () => { - it('will get unpublished files', async () => { - const lufiles = [ - { - diagnostics: [], - id: 'a', - relativePath: 'bot1/a.lu', - content: '', - intents: [], - }, - { - diagnostics: [], - id: 'b', - relativePath: 'bot1/b.lu', - content: '', - intents: [], - }, - ]; - - const luPublisher = new LuPublisher(botDir, storage); - await luPublisher.loadStatus(['bot1/a.lu', 'bot1/b.lu']); // relative path is key - - let files = luPublisher.getUnpublisedFiles(lufiles); - expect(files.length).toBe(2); - const curTime = Date.now(); - luPublisher.status['bot1/a.lu'].lastPublishTime = curTime; // assumming we publish a.lu - luPublisher.status['bot1/b.lu'].lastPublishTime = curTime; // and b.lu - files = luPublisher.getUnpublisedFiles(lufiles); - expect(files.length).toBe(0); - - await luPublisher.onFileChange('bot1/a.lu', FileUpdateType.UPDATE); - files = luPublisher.getUnpublisedFiles(lufiles); - expect(files.length).toBe(1); - }); -}); diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index bff3a64a80..b5acad1b41 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -156,7 +156,7 @@ export class LuPublisher { //add the lu file that are not in crossTrain config. luFiles.forEach(file => { - if (paths.indexOf(`${file.id}.lu`) === -1) { + if (!~paths.indexOf(`${file.id}.lu`)) { luConfig.models.push(Path.resolve(this.botDir, file.relativePath)); } }); From ae02ed271cedb5069b03cb6f8655266fc61b47bb Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 2 Mar 2020 19:30:59 +0800 Subject: [PATCH 09/14] fix unit test --- .../indexers/src/dialogUtils/extractIntentTriggers.ts | 2 +- Composer/packages/server/src/types/bf-lu.d.ts | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 Composer/packages/server/src/types/bf-lu.d.ts diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts index 8a56586425..2fe26064cb 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -11,7 +11,7 @@ function ExtractIntentTriggers(value: any): IIntentTrigger[] { const triggers: IIntentTrigger[] = []; const visitor: VisitorFunc = (path: string, value: any): boolean => { - if (value.$type === SDKTypes.OnIntent && value.actions[0]?.$type === SDKTypes.BeginDialog) { + if (value.$type === SDKTypes.OnIntent && value.actions?.[0]?.$type === SDKTypes.BeginDialog) { triggers.push({ intent: value.intent, dialog: value.actions[0].dialog, diff --git a/Composer/packages/server/src/types/bf-lu.d.ts b/Composer/packages/server/src/types/bf-lu.d.ts deleted file mode 100644 index cd8aa7ce49..0000000000 --- a/Composer/packages/server/src/types/bf-lu.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -/// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -declare module '@bfcomposer/bf-lu/lib/parser/lubuild' { - namespace crossTrain { - function train(input: any, intentName: any, configPath: any): any; - function writeFiles(fileIdToLuResourceMap: any, out: any): any; - } -} From 0aea283e183aadd2315e24688dede2da3733e75f Mon Sep 17 00:00:00 2001 From: leilzh Date: Tue, 3 Mar 2020 23:17:11 +0800 Subject: [PATCH 10/14] update the bf-lu --- .../src/dialogUtils/extractIntentTriggers.ts | 2 +- .../server/src/models/bot/botProject.ts | 2 +- .../server/src/models/bot/luPublisher.ts | 29 ++++++++------- Composer/packages/server/src/types/bf-lu.d.ts | 35 ------------------- 4 files changed, 18 insertions(+), 50 deletions(-) delete mode 100644 Composer/packages/server/src/types/bf-lu.d.ts diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts index 2fe26064cb..b3f0fd2126 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -11,7 +11,7 @@ function ExtractIntentTriggers(value: any): IIntentTrigger[] { const triggers: IIntentTrigger[] = []; const visitor: VisitorFunc = (path: string, value: any): boolean => { - if (value.$type === SDKTypes.OnIntent && value.actions?.[0]?.$type === SDKTypes.BeginDialog) { + if (value?.$type === SDKTypes.OnIntent && value.actions?.[0]?.$type === SDKTypes.BeginDialog) { triggers.push({ intent: value.intent, dialog: value.actions[0].dialog, diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index ca59180b2c..1071c0cd12 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -329,7 +329,7 @@ export class BotProject { } if (referred.length > 0) { - await this.luPublisher.setCrossTrainConfig(this.dialogs); + this.luPublisher.setCrossTrainConfig(this.dialogs); await this.luPublisher.publish(referred); } diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index 8de8a6a77e..c74085e9e7 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import isEqual from 'lodash/isEqual'; +import keys from 'lodash/keys'; import { LuFile, DialogInfo } from '@bfc/indexers'; import { Path } from './../../utility/path'; @@ -44,17 +45,10 @@ export class LuPublisher { try { await this._createGeneratedDir(); - await this.createCrossTrainConfig(); - //do cross train before publish await this._crossTrain(); - const config = await this._getConfig(luFiles); - if (config.models.length === 0) { - throw new Error('No luis file exist'); - } - - await this._runBuild(config); + await this._runBuild(luFiles); //remove the cross train result await this._cleanCrossTrain(); @@ -78,7 +72,7 @@ export class LuPublisher { } }; - public setCrossTrainConfig = async (dialogs: DialogInfo[]) => { + public setCrossTrainConfig = (dialogs: DialogInfo[]) => { // ToDo: create real tree for cross train. Now add this data to test the bf-lu const rootDialog = dialogs.find(dialog => dialog.isRoot); if (rootDialog?.intentTriggers.length) { @@ -87,7 +81,7 @@ export class LuPublisher { }; //write config to generated folder - public async createCrossTrainConfig() { + private async _createCrossTrainConfig() { await this.storage.writeFile( `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, JSON.stringify(this.crossTrainMapRule) @@ -123,8 +117,13 @@ export class LuPublisher { } } + private _needCrossTrain() { + return !!keys(this.crossTrainMapRule).length; + } + private async _crossTrain() { - //await this.createCrossTrainConfig(); + if (!this._needCrossTrain()) return; + await this._createCrossTrainConfig(); const result = await crossTrain.train( this.dialogsDir, '_Interuption', @@ -185,8 +184,11 @@ export class LuPublisher { luConfig.models = []; //add all lu file after cross train - const paths = await this.storage.glob('**/*.lu', this.interuptionFolderPath); - luConfig.models = paths.map(filePath => Path.join(this.interuptionFolderPath, filePath)); + let paths: string[] = []; + if (this._needCrossTrain()) { + paths = await this.storage.glob('**/*.lu', this.interuptionFolderPath); + luConfig.models = paths.map(filePath => Path.join(this.interuptionFolderPath, filePath)); + } //add the lu file that are not in crossTrain config. luFiles.forEach(file => { @@ -207,6 +209,7 @@ export class LuPublisher { }; private async _cleanCrossTrain() { + if (!this._needCrossTrain()) return; await this.storage.removeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`); await this._deleteDir(this.interuptionFolderPath); } diff --git a/Composer/packages/server/src/types/bf-lu.d.ts b/Composer/packages/server/src/types/bf-lu.d.ts deleted file mode 100644 index 1eff54c908..0000000000 --- a/Composer/packages/server/src/types/bf-lu.d.ts +++ /dev/null @@ -1,35 +0,0 @@ -/// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -declare module '@bfcomposer/bf-lu/lib/parser/lubuild' { - namespace luBuild { - class Builder { - private readonly handler: (input: string) => any; - - constructor(handler: any); - - loadContents( - files: string[], - culture: string, - suffix: string, - region: string, - stdin?: string - ): { luContents: any[]; recognizers: Map; multiRecognizers: Map; settings: any }; - - build( - luContents: any[], - recognizers: Map, - authoringKey: string, - region: string, - botName: string, - suffix: string, - fallbackLocale: string, - deleteOldVersion: boolean, - multiRecognizers: Map, - settings: any - ): any[]; - - writeDialogAssets(contents: any[], force: boolean, out: string): void; - } - } -} From 039c1fc5ae834bdeb345a9602bbab7e165bd048a Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 4 Mar 2020 21:47:51 +0800 Subject: [PATCH 11/14] update the function --- .../server/src/models/bot/luPublisher.ts | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index c74085e9e7..fc93c43a0b 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -11,14 +11,13 @@ import { ILuisConfig } from './interface'; import log from './../../logger'; // eslint-disable-next-line @typescript-eslint/no-var-requires -const crossTrain = require('@bfcomposer/bf-lu/lib/parser/cross-train/cross-train.js'); +const crossTrainer = require('@bfcomposer/bf-lu/lib/parser/cross-train/crossTrainer.js'); // eslint-disable-next-line @typescript-eslint/no-var-requires const luBuild = require('@bfcomposer/bf-lu/lib/parser/lubuild/builder.js'); const DIALOGS_FOLDER = 'ComposerDialogs'; const GENERATEDFOLDER = 'generated'; const INTERUPTION = 'interuption'; -const CROSS_TRAIN_CONFIG = 'mapping_rules.json'; export class LuPublisher { public botDir: string; @@ -46,7 +45,7 @@ export class LuPublisher { await this._createGeneratedDir(); //do cross train before publish - await this._crossTrain(); + await this._crossTrain(luFiles); await this._runBuild(luFiles); @@ -80,13 +79,13 @@ export class LuPublisher { } }; - //write config to generated folder - private async _createCrossTrainConfig() { - await this.storage.writeFile( - `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, - JSON.stringify(this.crossTrainMapRule) - ); - } + // //write config to generated folder + // private async _createCrossTrainConfig() { + // await this.storage.writeFile( + // `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, + // JSON.stringify(this.crossTrainMapRule) + // ); + // } private _createPath(dialogPath: string) { const path = dialogPath.replace('.dialog', '.lu'); @@ -95,6 +94,7 @@ export class LuPublisher { return relativePath; } + //generate the cross-train config private _createTree(dialog: DialogInfo, dialogs: DialogInfo[]) { let result = {}; const key = this._createPath(dialog.relativePath); @@ -121,15 +121,29 @@ export class LuPublisher { return !!keys(this.crossTrainMapRule).length; } - private async _crossTrain() { + private async _crossTrain(luFiles: LuFile[]) { if (!this._needCrossTrain()) return; - await this._createCrossTrainConfig(); - const result = await crossTrain.train( - this.dialogsDir, - '_Interuption', - `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}` + const luContents = luFiles.map(file => { + return { content: file.content, path: Path.join(this.botDir, file.relativePath) }; + }); + const configContent = crossTrainer.getConfigObject( + { path: this.interuptionFolderPath, content: JSON.stringify(this.crossTrainMapRule) }, + '_Interuption' ); - await crossTrain.writeFiles(result.luResult, this.interuptionFolderPath); + const result = await crossTrainer.crossTrain(luContents, [], configContent); + + await this._writeFiles(result.luResult); + } + + private async _writeFiles(crossTrainResult) { + if (!(await this.storage.exists(this.interuptionFolderPath))) { + await this.storage.mkDir(this.interuptionFolderPath); + } + for (const key of crossTrainResult.keys()) { + const fileName = Path.basename(key); + const newFileId = Path.join(this.interuptionFolderPath, fileName); + await this.storage.writeFile(newFileId, crossTrainResult.get(key).Content); + } } private async _runBuild(luFiles: LuFile[]) { @@ -190,7 +204,7 @@ export class LuPublisher { luConfig.models = paths.map(filePath => Path.join(this.interuptionFolderPath, filePath)); } - //add the lu file that are not in crossTrain config. + //add the lu file that are not in interuption folder. luFiles.forEach(file => { if (!~paths.indexOf(`${file.id}.lu`)) { luConfig.models.push(Path.resolve(this.botDir, file.relativePath)); @@ -210,7 +224,6 @@ export class LuPublisher { private async _cleanCrossTrain() { if (!this._needCrossTrain()) return; - await this.storage.removeFile(`${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`); await this._deleteDir(this.interuptionFolderPath); } } From 67cc964f5433d9bc0b4a0e1376259619ec9cd26a Mon Sep 17 00:00:00 2001 From: leilzh Date: Thu, 5 Mar 2020 17:12:13 +0800 Subject: [PATCH 12/14] update the crosstrain config --- Composer/packages/client/package.json | 2 +- Composer/packages/lib/indexers/package.json | 2 +- .../src/dialogUtils/extractIntentTriggers.ts | 24 ++- .../lib/indexers/src/dialogUtils/types.ts | 2 +- Composer/packages/server/package.json | 2 +- .../server/src/models/bot/botProject.ts | 2 +- .../server/src/models/bot/luPublisher.ts | 95 +++++++----- .../language-understanding/package.json | 4 +- Composer/yarn.lock | 144 +++++++++++++++--- 9 files changed, 200 insertions(+), 77 deletions(-) diff --git a/Composer/packages/client/package.json b/Composer/packages/client/package.json index e16b0ecc95..842169d718 100644 --- a/Composer/packages/client/package.json +++ b/Composer/packages/client/package.json @@ -20,7 +20,7 @@ "@bfc/extensions": "*", "@bfc/indexers": "*", "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.2.7", + "@bfcomposer/bf-lu": "^1.2.9", "@emotion/core": "^10.0.7", "@reach/router": "^1.2.1", "axios": "^0.18.0", diff --git a/Composer/packages/lib/indexers/package.json b/Composer/packages/lib/indexers/package.json index e8ff941b99..b0102f5096 100644 --- a/Composer/packages/lib/indexers/package.json +++ b/Composer/packages/lib/indexers/package.json @@ -29,7 +29,7 @@ }, "dependencies": { "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.2.7", + "@bfcomposer/bf-lu": "^1.2.9", "botbuilder-lg": "^4.8.0-preview.106823", "botframework-expressions": "^4.8.0-preview.106476", "lodash": "^4.17.15" diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts index b3f0fd2126..ef010d02be 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -12,10 +12,26 @@ function ExtractIntentTriggers(value: any): IIntentTrigger[] { const visitor: VisitorFunc = (path: string, value: any): boolean => { if (value?.$type === SDKTypes.OnIntent && value.actions?.[0]?.$type === SDKTypes.BeginDialog) { - triggers.push({ - intent: value.intent, - dialog: value.actions[0].dialog, - }); + if (value.intent) { + const dialogs: string[] = []; + + const visitor: VisitorFunc = (path: string, value: any): boolean => { + if (value?.$type === SDKTypes.BeginDialog) { + if (value.dialog) { + dialogs.push(value.dialog); + } + return true; + } + return false; + }; + JsonWalk('$', value, visitor); + if (dialogs.length) { + triggers.push({ + intent: value.intent, + dialogs, + }); + } + } return true; } return false; diff --git a/Composer/packages/lib/indexers/src/dialogUtils/types.ts b/Composer/packages/lib/indexers/src/dialogUtils/types.ts index 0eb1b88f4b..974ccef47f 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/types.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/types.ts @@ -25,7 +25,7 @@ export interface IExpressionProperties { export interface IIntentTrigger { intent: string; - dialog: string; + dialogs: string[]; } export type CheckerFunc = (path: string, value: any, type: string, schema: any) => Diagnostic[] | null; // error msg diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index 1fcb86e115..aab1124761 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -57,7 +57,7 @@ "@bfc/lg-languageserver": "*", "@bfc/lu-languageserver": "*", "@bfc/shared": "*", - "@bfcomposer/bf-lu": "^1.2.7", + "@bfcomposer/bf-lu": "^1.2.9", "archiver": "^3.0.0", "axios": "^0.18.0", "azure-storage": "^2.10.3", diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index 1071c0cd12..036b7b6d64 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -329,7 +329,7 @@ export class BotProject { } if (referred.length > 0) { - this.luPublisher.setCrossTrainConfig(this.dialogs); + this.luPublisher.createCrossTrainConfig(this.dialogs, referred); await this.luPublisher.publish(referred); } diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index fc93c43a0b..d0907334de 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -19,6 +19,13 @@ const DIALOGS_FOLDER = 'ComposerDialogs'; const GENERATEDFOLDER = 'generated'; const INTERUPTION = 'interuption'; +interface ICrossTrainConfig { + rootIds: string[]; + triggerRules: { [key: string]: any }; + intentName: string; + verbose: boolean; +} + export class LuPublisher { public botDir: string; public dialogsDir: string; @@ -26,7 +33,12 @@ export class LuPublisher { public interuptionFolderPath: string; public storage: IFileStorage; public config: ILuisConfig | null = null; - public crossTrainMapRule: { [key: string]: string } = {}; + public crossTrainMapRule: ICrossTrainConfig = { + rootIds: [], + triggerRules: {}, + intentName: '_Interruption', + verbose: true, + }; private builder = new luBuild.Builder(message => { log(message); @@ -71,44 +83,48 @@ export class LuPublisher { } }; - public setCrossTrainConfig = (dialogs: DialogInfo[]) => { - // ToDo: create real tree for cross train. Now add this data to test the bf-lu - const rootDialog = dialogs.find(dialog => dialog.isRoot); - if (rootDialog?.intentTriggers.length) { - this.crossTrainMapRule = this._createTree(rootDialog, dialogs); - } - }; + //generate the cross-train config + public createCrossTrainConfig = (dialogs: DialogInfo[], luFiles: LuFile[]) => { + const triggerRules = {}; + const countMap = {}; - // //write config to generated folder - // private async _createCrossTrainConfig() { - // await this.storage.writeFile( - // `${this.generatedFolderPath}/${CROSS_TRAIN_CONFIG}`, - // JSON.stringify(this.crossTrainMapRule) - // ); - // } - - private _createPath(dialogPath: string) { - const path = dialogPath.replace('.dialog', '.lu'); - const absolutePath = Path.join(this.botDir, path); - const relativePath = Path.relative(this.generatedFolderPath, absolutePath); - return relativePath; - } + //map all referred lu files + luFiles.forEach(file => { + countMap[file.id] = 0; + }); - //generate the cross-train config - private _createTree(dialog: DialogInfo, dialogs: DialogInfo[]) { - let result = {}; - const key = this._createPath(dialog.relativePath); - dialog.intentTriggers.forEach(temp => { - const target = dialogs.find(dialog => dialog.id === temp.dialog); - if (target && target.content?.recognizer) { - if (!result[key]) result[key] = { triggers: {} }; - result[key].triggers[temp.intent] = this._createPath(target.relativePath); - result = { ...result, ...this._createTree(target, dialogs) }; + dialogs.forEach(dialog => { + const { intentTriggers } = dialog; + const fileId = this._createConfigId(dialog.id); + if (intentTriggers.length) { + intentTriggers.forEach(item => { + const used = item.dialogs.filter(dialog => { + if (typeof countMap[dialog] === 'number') { + countMap[dialog]++; + return true; + } + return false; + }); + if (used.length) { + const result = used.reduce((result, temp) => { + const id = this._createConfigId(temp); + result[id] = item.intent; + return result; + }, {}); + triggerRules[fileId] = { ...triggerRules[fileId], ...result }; + } + }); } }); - if (dialog.isRoot && result[key]) result[key].rootDialog = true; - return result; + this.crossTrainMapRule.rootIds = keys(countMap) + .filter(key => (countMap[key] === 0 || key === 'Main') && triggerRules[this._createConfigId(key)]) + .map(item => this._createConfigId(item)); + this.crossTrainMapRule.triggerRules = triggerRules; + }; + + private _createConfigId(fileId) { + return `${fileId}.lu`; } private async _createGeneratedDir() { @@ -118,19 +134,16 @@ export class LuPublisher { } private _needCrossTrain() { - return !!keys(this.crossTrainMapRule).length; + return !!this.crossTrainMapRule.rootIds.length; } private async _crossTrain(luFiles: LuFile[]) { if (!this._needCrossTrain()) return; const luContents = luFiles.map(file => { - return { content: file.content, path: Path.join(this.botDir, file.relativePath) }; + return { content: file.content, id: this._createConfigId(file.id) }; }); - const configContent = crossTrainer.getConfigObject( - { path: this.interuptionFolderPath, content: JSON.stringify(this.crossTrainMapRule) }, - '_Interuption' - ); - const result = await crossTrainer.crossTrain(luContents, [], configContent); + + const result = await crossTrainer.crossTrain(luContents, [], this.crossTrainMapRule); await this._writeFiles(result.luResult); } diff --git a/Composer/packages/tools/language-servers/language-understanding/package.json b/Composer/packages/tools/language-servers/language-understanding/package.json index c151e7a31f..f74756c548 100644 --- a/Composer/packages/tools/language-servers/language-understanding/package.json +++ b/Composer/packages/tools/language-servers/language-understanding/package.json @@ -18,8 +18,8 @@ "start:server": "cd demo && ts-node ./src/server.ts" }, "dependencies": { - "@microsoft/bf-cli-command": "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz", - "@bfcomposer/bf-lu": "^1.2.7", + "@microsoft/bf-cli-command": "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.1.tgz", + "@bfcomposer/bf-lu": "^1.2.9", "@types/node": "^12.0.4", "express": "^4.15.2", "monaco-editor-core": "^0.17.0", diff --git a/Composer/yarn.lock b/Composer/yarn.lock index ee27ebb007..82eb4a7ab4 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -2120,10 +2120,10 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@bfcomposer/bf-lu@^1.2.7": - version "1.2.7" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.7.tgz#7bdcec5660c894164cc17f7065e76981bd57bcea" - integrity sha1-e9zsVmDIlBZMwX9wZedpgb1XvOo= +"@bfcomposer/bf-lu@^1.2.9": + version "1.2.9" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@bfcomposer/bf-lu/-/@bfcomposer/bf-lu-1.2.9.tgz#b2fec0d8f1c3d4d0e526a5bdbf9e5c9b3ab5b71d" + integrity sha1-sv7A2PHD1NDlJqW9v55cmzq1tx0= dependencies: "@azure/cognitiveservices-luis-authoring" "3.0.1" "@azure/ms-rest-azure-js" "2.0.1" @@ -2134,6 +2134,7 @@ delay "^4.3.0" fs-extra "^8.1.0" get-stdin "^6.0.0" + globby "^10.0.1" intercept-stdout "^0.1.2" lodash "^4.17.15" node-fetch "^2.1.2" @@ -2618,19 +2619,19 @@ "@types/istanbul-reports" "^1.1.1" "@types/yargs" "^13.0.0" -"@microsoft/bf-cli-command@https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz": - version "1.0.0" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.0.tgz#0d681e93690c4e872bf1181a3ff02f094830929e" - integrity sha1-DWgek2kMTocr8RgaP/AvCUgwkp4= +"@microsoft/bf-cli-command@https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.1.tgz": + version "1.0.1" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-cli-command/-/@microsoft/bf-cli-command-1.0.1.tgz#5c7f650015f839d80a7739bc70f8cd5fa90a1e48" dependencies: - "@oclif/command" "~1.5.13" - "@oclif/config" "~1.12.12" + "@oclif/command" "~1.5.19" + "@oclif/config" "~1.13.3" "@oclif/errors" "~1.2.2" applicationinsights "^1.0.8" chalk "2.4.1" cli-ux "~4.9.3" debug "^4.1.1" fs-extra "^7.0.1" + tslib "~1.10.0" "@microsoft/load-themed-styles@^1.7.13": version "1.9.5" @@ -2650,15 +2651,36 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@nodelib/fs.scandir@2.1.3": + version "2.1.3" + resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz#3a582bdb53804c6ba6d146579c46e52130cf4a3b" + integrity sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw== + dependencies: + "@nodelib/fs.stat" "2.0.3" + run-parallel "^1.1.9" + +"@nodelib/fs.stat@2.0.3", "@nodelib/fs.stat@^2.0.2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz#34dc5f4cabbc720f4e60f75a747e7ecd6c175bd3" + integrity sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA== + "@nodelib/fs.stat@^1.1.2": version "1.1.3" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw== -"@oclif/command@^1.5.13", "@oclif/command@~1.5.13": +"@nodelib/fs.walk@^1.2.3": + version "1.2.4" + resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz#011b9202a70a6366e436ca5c065844528ab04976" + integrity sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ== + dependencies: + "@nodelib/fs.scandir" "2.1.3" + fastq "^1.6.0" + +"@oclif/command@^1.5.13", "@oclif/command@~1.5.19": version "1.5.19" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/command/-/@oclif/command-1.5.19.tgz#13f472450eb83bd6c6871a164c03eadb5e1a07ed" - integrity sha1-E/RyRQ64O9bGhxoWTAPq214aB+0= + resolved "https://registry.yarnpkg.com/@oclif/command/-/command-1.5.19.tgz#13f472450eb83bd6c6871a164c03eadb5e1a07ed" + integrity sha512-6+iaCMh/JXJaB2QWikqvGE9//wLEVYYwZd5sud8aLoLKog1Q75naZh2vlGVtg5Mq/NqpqGQvdIjJb3Bm+64AUQ== dependencies: "@oclif/config" "^1" "@oclif/errors" "^1.2.2" @@ -2667,7 +2689,7 @@ debug "^4.1.1" semver "^5.6.0" -"@oclif/config@^1": +"@oclif/config@^1", "@oclif/config@~1.13.3": version "1.13.3" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/config/-/@oclif/config-1.13.3.tgz#1b13e18d0e4242ddbd9cbd100f0eec819aa2bf8c" integrity sha1-GxPhjQ5CQt29nL0QDw7sgZqiv4w= @@ -2676,14 +2698,6 @@ debug "^4.1.1" tslib "^1.9.3" -"@oclif/config@~1.12.12": - version "1.12.12" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/config/-/@oclif/config-1.12.12.tgz#09e6628fe454214bbd7630ae18d2bee0db6bd533" - integrity sha1-CeZij+RUIUu9djCuGNK+4Ntr1TM= - dependencies: - debug "^4.1.1" - tslib "^1.9.3" - "@oclif/errors@^1.2.2", "@oclif/errors@~1.2.2": version "1.2.2" resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@oclif/errors/-/@oclif/errors-1.2.2.tgz#9d8f269b15f13d70aa93316fed7bebc24688edc2" @@ -4166,6 +4180,11 @@ array-union@^1.0.1, array-union@^1.0.2: dependencies: array-uniq "^1.0.1" +array-union@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== + array-uniq@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" @@ -7273,6 +7292,13 @@ dir-glob@^2.2.1: dependencies: path-type "^3.0.0" +dir-glob@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== + dependencies: + path-type "^4.0.0" + dns-equal@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d" @@ -8213,6 +8239,18 @@ fast-glob@^2.0.2, fast-glob@^2.2.6: merge2 "^1.2.3" micromatch "^3.1.10" +fast-glob@^3.0.3: + version "3.2.2" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.2.tgz#ade1a9d91148965d4bf7c51f72e1ca662d32e63d" + integrity sha512-UDV82o4uQyljznxwMxyVRJgZZt3O5wENYojjzbaGEGZgeOxkLFf+V4cnUD+krzb2F72E18RhamkMZ7AdeggF7A== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.0" + merge2 "^1.3.0" + micromatch "^4.0.2" + picomatch "^2.2.1" + fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" @@ -8223,6 +8261,13 @@ fast-levenshtein@~2.0.4, fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= +fastq@^1.6.0: + version "1.6.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.6.1.tgz#4570c74f2ded173e71cf0beb08ac70bb85826791" + integrity sha512-mpIH5sKYueh3YyeJwqtVo8sORi0CgtmkVbK6kZStpQlZBYQuTzG2CZ7idSiJuA7bY0SFCWUc5WIs+oYumGCQNw== + dependencies: + reusify "^1.0.4" + faye-websocket@^0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4" @@ -8822,7 +8867,7 @@ glob-parent@^3.1.0: is-glob "^3.1.0" path-dirname "^1.0.0" -glob-parent@^5.0.0: +glob-parent@^5.0.0, glob-parent@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.0.tgz#5f4c1d1e748d30cd73ad2944b3577a81b081e8c2" integrity sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw== @@ -8969,6 +9014,20 @@ globby@8.0.2: pify "^3.0.0" slash "^1.0.0" +globby@^10.0.1: + version "10.0.2" + resolved "https://registry.yarnpkg.com/globby/-/globby-10.0.2.tgz#277593e745acaa4646c3ab411289ec47a0392543" + integrity sha512-7dUi7RvCoT/xast/o/dLN53oqND4yk0nsHkhRgn9w65C4PofCLOoJ39iSOg+qVDdWQPIEj+eszMHQ+aLVwwQSg== + dependencies: + "@types/glob" "^7.1.1" + array-union "^2.1.0" + dir-glob "^3.0.1" + fast-glob "^3.0.3" + glob "^7.1.3" + ignore "^5.1.1" + merge2 "^1.2.3" + slash "^3.0.0" + globby@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-6.1.0.tgz#f5a6d70e8395e21c858fb0489d64df02424d506c" @@ -9525,6 +9584,11 @@ ignore@^4.0.3, ignore@^4.0.6: resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== +ignore@^5.1.1: + version "5.1.4" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.4.tgz#84b7b3dbe64552b6ef0eca99f6743dbec6d97adf" + integrity sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A== + immer@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" @@ -11695,6 +11759,11 @@ merge2@^1.2.3: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.2.3.tgz#7ee99dbd69bb6481689253f018488a1b902b0ed5" integrity sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA== +merge2@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.3.0.tgz#5b366ee83b2f1582c48f87e47cf1a9352103ca81" + integrity sha512-2j4DAdlBOkiSZIsaXk4mTE3sRS02yBHAtfy127xRV3bQUFqXkjHCHLW6Scv7DwNRbIWNHH8zpnz9zMaKXIdvYw== + methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" @@ -11724,7 +11793,7 @@ micromatch@^3.0.4, micromatch@^3.1.10, micromatch@^3.1.4, micromatch@^3.1.8: snapdragon "^0.8.1" to-regex "^3.0.2" -micromatch@^4.0.0: +micromatch@^4.0.0, micromatch@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.2.tgz#4fcb0999bf9fbc2fcbdd212f6d629b9a56c39259" integrity sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q== @@ -13056,6 +13125,11 @@ path-type@^3.0.0: dependencies: pify "^3.0.0" +path-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== + pbkdf2@^3.0.3: version "3.0.17" resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.0.17.tgz#976c206530617b14ebb32114239f7b09336e93a6" @@ -13087,6 +13161,11 @@ picomatch@^2.0.5: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.0.7.tgz#514169d8c7cd0bdbeecc8a2609e34a7163de69f6" integrity sha512-oLHIdio3tZ0qH76NybpeneBhYVj0QFTfXEFTc/B3zKQspYfYYkWYgFsmzo+4kvId/bQRcNkVeguI3y+CD22BtA== +picomatch@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.1.tgz#21bac888b6ed8601f831ce7816e335bc779f0a4a" + integrity sha512-ISBaA8xQNmwELC7eOjqFKMESB2VIqt4PPDD0nsS95b/9dZXvVKOlz9keMSnoGGKcOHXfTvDD6WMaRoSc9UuhRA== + pify@^2.0.0, pify@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" @@ -14862,6 +14941,11 @@ retry@^0.12.0: resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= +reusify@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== + rgb-regex@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1" @@ -14911,6 +14995,11 @@ run-node@^1.0.0: resolved "https://registry.yarnpkg.com/run-node/-/run-node-1.0.0.tgz#46b50b946a2aa2d4947ae1d886e9856fd9cabe5e" integrity sha512-kc120TBlQ3mih1LSzdAJXo4xn/GWS2ec0l3S+syHDXP9uRr0JAT8Qd3mdMuyjqCzeZktgP3try92cEgf9Nks8A== +run-parallel@^1.1.9: + version "1.1.9" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.1.9.tgz#c9dd3a7cf9f4b2c4b6244e173a6ed866e61dd679" + integrity sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q== + run-queue@^1.0.0, run-queue@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47" @@ -15275,6 +15364,11 @@ slash@^2.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44" integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A== +slash@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" + integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== + slice-ansi@0.0.4: version "0.0.4" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" @@ -16384,7 +16478,7 @@ tsconfig-paths@^3.4.0: minimist "^1.2.0" strip-bom "^3.0.0" -tslib@^1.10.0, tslib@^1.9.2, tslib@^1.9.3: +tslib@^1.10.0, tslib@^1.9.2, tslib@^1.9.3, tslib@~1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== From faf96112f6fa343a155d48912bb12b02ee89ed28 Mon Sep 17 00:00:00 2001 From: leilzh Date: Thu, 5 Mar 2020 17:36:00 +0800 Subject: [PATCH 13/14] fix lint --- .../obiformeditor/src/Form/fields/PromptField/UserInput.tsx | 2 +- .../obiformeditor/src/Form/widgets/LuEditorWidget.tsx | 1 + .../lib/indexers/src/dialogUtils/extractIntentTriggers.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Composer/packages/extensions/obiformeditor/src/Form/fields/PromptField/UserInput.tsx b/Composer/packages/extensions/obiformeditor/src/Form/fields/PromptField/UserInput.tsx index 97553ed0f0..d2550e546d 100644 --- a/Composer/packages/extensions/obiformeditor/src/Form/fields/PromptField/UserInput.tsx +++ b/Composer/packages/extensions/obiformeditor/src/Form/fields/PromptField/UserInput.tsx @@ -8,6 +8,7 @@ import { FieldProps } from '@bfcomposer/react-jsonschema-form'; import formatMessage from 'format-message'; import { JSONSchema6 } from 'json-schema'; import { SDKTypes, MicrosoftInputDialog, ChoiceInput, ConfirmInput } from '@bfc/shared'; +import { DialogInfo } from '@bfc/indexers/lib/type'; import { TextWidget, SelectWidget } from '../../widgets'; import { LuEditorWidget } from '../../widgets/LuEditorWidget'; @@ -16,7 +17,6 @@ import { field } from './styles'; import { GetSchema, PromptFieldChangeHandler } from './types'; import { ChoiceInputSettings } from './ChoiceInput'; import { ConfirmInputSettings } from './ConfirmInput'; -import { DialogInfo } from '@bfc/indexers/lib/type'; const getOptions = (enumSchema: JSONSchema6) => { if (!enumSchema || !enumSchema.enum || !Array.isArray(enumSchema.enum)) { diff --git a/Composer/packages/extensions/obiformeditor/src/Form/widgets/LuEditorWidget.tsx b/Composer/packages/extensions/obiformeditor/src/Form/widgets/LuEditorWidget.tsx index c551e5bdab..dba358ad9f 100644 --- a/Composer/packages/extensions/obiformeditor/src/Form/widgets/LuEditorWidget.tsx +++ b/Composer/packages/extensions/obiformeditor/src/Form/widgets/LuEditorWidget.tsx @@ -9,6 +9,7 @@ import { LuFile, filterSectionDiagnostics } from '@bfc/indexers'; import { LuIntentSection } from '@bfc/shared'; import { FormContext } from '../types'; + import { WidgetLabel } from './WidgetLabel'; interface LuEditorWidgetProps { diff --git a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts index ef010d02be..ff5e4bf01d 100644 --- a/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts +++ b/Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts @@ -11,7 +11,7 @@ function ExtractIntentTriggers(value: any): IIntentTrigger[] { const triggers: IIntentTrigger[] = []; const visitor: VisitorFunc = (path: string, value: any): boolean => { - if (value?.$type === SDKTypes.OnIntent && value.actions?.[0]?.$type === SDKTypes.BeginDialog) { + if (value?.$type === SDKTypes.OnIntent) { if (value.intent) { const dialogs: string[] = []; From 1825c90bfa994a2c58fd5d5cdfb02aea44ac58ad Mon Sep 17 00:00:00 2001 From: leilzh Date: Thu, 5 Mar 2020 23:13:11 +0800 Subject: [PATCH 14/14] add some comments --- .../server/src/models/bot/luPublisher.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Composer/packages/server/src/models/bot/luPublisher.ts b/Composer/packages/server/src/models/bot/luPublisher.ts index d0907334de..67c589a260 100644 --- a/Composer/packages/server/src/models/bot/luPublisher.ts +++ b/Composer/packages/server/src/models/bot/luPublisher.ts @@ -84,6 +84,29 @@ export class LuPublisher { }; //generate the cross-train config + /* the config is like + { + rootIds: [ + 'main.lu', + 'main.fr-fr.lu' + ], + triggerRules: { + 'main.lu': { + 'dia1.lu': 'dia1_trigger', + 'dia2.lu': 'dia2_trigger' + }, + 'dia2.lu': { + 'dia3.lu': 'dia3_trigger', + 'dia4.lu': 'dia4_trigger' + }, + 'main.fr-fr.lu': { + 'dia1.fr-fr.lu': 'dia1_trigger' + } + }, + intentName: '_Interruption', + verbose: true + } + */ public createCrossTrainConfig = (dialogs: DialogInfo[], luFiles: LuFile[]) => { const triggerRules = {}; const countMap = {}; @@ -97,6 +120,7 @@ export class LuPublisher { const { intentTriggers } = dialog; const fileId = this._createConfigId(dialog.id); if (intentTriggers.length) { + //find the trigger's dialog that use a recognizer intentTriggers.forEach(item => { const used = item.dialogs.filter(dialog => { if (typeof countMap[dialog] === 'number') {