diff --git a/Composer/packages/client/__tests__/utils/buildUtil.test.ts b/Composer/packages/client/__tests__/utils/buildUtil.test.ts index ce3e9fb67d..1a19bf9e16 100644 --- a/Composer/packages/client/__tests__/utils/buildUtil.test.ts +++ b/Composer/packages/client/__tests__/utils/buildUtil.test.ts @@ -80,14 +80,11 @@ describe('createCrossTrainConfig', () => { { id: 'dia5.en-us' }, { id: 'dia6.en-us' }, ]; - const config = createCrossTrainConfig(dialogs as DialogInfo[], luFiles as LuFile[]); - expect(config.rootIds.length).toEqual(1); - expect(config.rootIds[0]).toEqual('main.en-us.lu'); - expect(config.triggerRules['main.en-us.lu'].dia1_trigger).toEqual('dia1.en-us.lu'); - expect(config.triggerRules['main.en-us.lu'].no_dialog).toEqual(''); - expect(config.triggerRules['main.en-us.lu'].dia1_trigger).toEqual('dia1.en-us.lu'); - expect(config.triggerRules['main.en-us.lu'].dias_trigger.length).toBe(2); - expect(config.triggerRules['dia1.en-us.lu'].dia3_trigger).toEqual('dia3.en-us.lu'); - expect(config.triggerRules['dia1.en-us.lu']['dia4.en-us.lu']).toBeUndefined(); + const config = createCrossTrainConfig(dialogs as DialogInfo[], luFiles as LuFile[], ['en-us']); + expect(config['main.en-us.lu'].rootDialog).toBeTruthy(); + expect(config['main.en-us.lu'].triggers.dia1_trigger[0]).toEqual('dia1.en-us.lu'); + expect(config['main.en-us.lu'].triggers.no_dialog.length).toEqual(0); + expect(config['main.en-us.lu'].triggers.dia2_trigger[0]).toEqual('dia2.en-us.lu'); + expect(config['main.en-us.lu'].triggers.dias_trigger.length).toBe(2); }); }); diff --git a/Composer/packages/client/src/components/TestController/TestController.tsx b/Composer/packages/client/src/components/TestController/TestController.tsx index 27549b7274..1bdd05e62d 100644 --- a/Composer/packages/client/src/components/TestController/TestController.tsx +++ b/Composer/packages/client/src/components/TestController/TestController.tsx @@ -9,6 +9,7 @@ import { PrimaryButton } from 'office-ui-fabric-react/lib/Button'; import formatMessage from 'format-message'; import { useRecoilValue } from 'recoil'; import { IConfig, IPublishConfig, defaultPublishConfig } from '@bfc/shared'; +import { useRecognizerConfig } from '@bfc/extension-client'; import { botEndpointsState, @@ -67,6 +68,7 @@ export const TestController: React.FC<{ projectId: string }> = (props) => { const settings = useRecoilValue(settingsState(projectId)); const qnaFiles = useRecoilValue(qnaFilesState(projectId)); const botLoadErrorMsg = useRecoilValue(botLoadErrorState(projectId)); + const { recognizers } = useRecognizerConfig(); const botEndpoints = useRecoilValue(botEndpointsState); const { @@ -156,12 +158,18 @@ export const TestController: React.FC<{ projectId: string }> = (props) => { setBotStatus(BotStatus.publishing, projectId); dismissDialog(); const { luis, qna } = config; + const recognizerTypes = dialogs.reduce((result, file) => { + const recognizer = recognizers.filter((r) => r.isSelected && r.isSelected(file.content.recognizer)); + result[file.id] = recognizer[0]?.id || ''; + return result; + }, {}); + await setSettings(projectId, { ...settings, luis: luis, qna: Object.assign({}, settings.qna, qna), }); - await build(luis, qna, projectId); + await build(luis, qna, recognizerTypes, projectId); } async function handleLoadBot() { diff --git a/Composer/packages/client/src/components/Toolbar.tsx b/Composer/packages/client/src/components/Toolbar.tsx index c50fde11f0..11ab4b1913 100644 --- a/Composer/packages/client/src/components/Toolbar.tsx +++ b/Composer/packages/client/src/components/Toolbar.tsx @@ -3,11 +3,17 @@ /** @jsx jsx */ import { jsx, css } from '@emotion/core'; -import { Fragment } from 'react'; +import { Fragment, useMemo } from 'react'; import formatMessage from 'format-message'; import { NeutralColors } from '@uifabric/fluent-theme'; import { ActionButton, CommandButton } from 'office-ui-fabric-react/lib/Button'; import { IContextualMenuProps, IIconProps } from 'office-ui-fabric-react/lib'; +import { EditorExtension, mergePluginConfigs, PluginConfig } from '@bfc/extension-client'; +import { useRecoilValue } from 'recoil'; + +import plugins from '../plugins'; +import { currentProjectIdState, schemasState } from '../recoilModel'; +import { useShell } from '../shell/useShell'; // -------------------- Styles -------------------- // @@ -98,6 +104,15 @@ type ToolbarProps = { // fabric-ui IButtonProps interface} export const Toolbar = (props: ToolbarProps) => { const { toolbarItems = [], ...rest } = props; + const projectId = useRecoilValue(currentProjectIdState); + const schemas = useRecoilValue(schemasState(projectId)); + const shellForPropertyEditor = useShell('DesignPage', projectId); + + const pluginConfig: PluginConfig = useMemo(() => { + const sdkUISchema = schemas?.ui?.content ?? {}; + const userUISchema = schemas?.uiOverrides?.content ?? {}; + return mergePluginConfigs({ uiSchema: sdkUISchema }, plugins, { uiSchema: userUISchema }); + }, [schemas?.ui?.content, schemas?.uiOverrides?.content]); const left: IToolbarItem[] = []; const right: IToolbarItem[] = []; @@ -113,9 +128,11 @@ export const Toolbar = (props: ToolbarProps) => { } return ( -
-
{left.map(renderItemList)}
-
{right.map(renderItemList)}
-
+ +
+
{left.map(renderItemList)}
+
{right.map(renderItemList)}
+
+
); }; diff --git a/Composer/packages/client/src/recoilModel/dispatchers/builder.ts b/Composer/packages/client/src/recoilModel/dispatchers/builder.ts index 81c92cca33..c2af7d8136 100644 --- a/Composer/packages/client/src/recoilModel/dispatchers/builder.ts +++ b/Composer/packages/client/src/recoilModel/dispatchers/builder.ts @@ -12,6 +12,7 @@ import httpClient from '../../utils/httpUtil'; import luFileStatusStorage from '../../utils/luFileStatusStorage'; import qnaFileStatusStorage from '../../utils/qnaFileStatusStorage'; import { luFilesState, qnaFilesState, dialogsState, botStatusState, botLoadErrorState } from '../atoms'; +import { settingsState } from '../atoms/botState'; const checkEmptyQuestionOrAnswerInQnAFile = (sections) => { return sections.some((s) => !s.Answer || s.Questions.some((q) => !q.content)); @@ -22,13 +23,14 @@ export const builderDispatcher = () => { ({ set, snapshot }: CallbackInterface) => async ( luisConfig: ILuisConfig, qnaConfig: IQnAConfig, + recognizerTypes: { [fileName: string]: string }, projectId: string ) => { const dialogs = await snapshot.getPromise(dialogsState(projectId)); const luFiles = await snapshot.getPromise(luFilesState(projectId)); const qnaFiles = await snapshot.getPromise(qnaFilesState(projectId)); + const settings = await snapshot.getPromise(settingsState(projectId)); const referredLuFiles = luUtil.checkLuisBuild(luFiles, dialogs); - const errorMsg = qnaFiles.reduce( (result, file) => { if ( @@ -48,15 +50,15 @@ export const builderDispatcher = () => { return; } try { - //TODO crosstrain should add locale - const crossTrainConfig = buildUtil.createCrossTrainConfig(dialogs, referredLuFiles); + const crossTrainConfig = buildUtil.createCrossTrainConfig(dialogs, referredLuFiles, settings.languages); await httpClient.post(`/projects/${projectId}/build`, { luisConfig, qnaConfig, projectId, crossTrainConfig, - luFiles: referredLuFiles.map((file) => file.id), - qnaFiles: qnaFiles.map((file) => file.id), + recognizerTypes, + luFiles: referredLuFiles.map((file) => ({ id: file.id, isEmpty: file.empty })), + qnaFiles: qnaFiles.map((file) => ({ id: file.id, isEmpty: !file.qnaSections.length })), }); luFileStatusStorage.publishAll(projectId); qnaFileStatusStorage.publishAll(projectId); diff --git a/Composer/packages/client/src/utils/buildUtil.ts b/Composer/packages/client/src/utils/buildUtil.ts index 29958f92e0..8769f7ff50 100644 --- a/Composer/packages/client/src/utils/buildUtil.ts +++ b/Composer/packages/client/src/utils/buildUtil.ts @@ -2,137 +2,45 @@ // Licensed under the MIT License. import { DialogInfo, LuFile } from '@bfc/shared'; -import keys from 'lodash/keys'; import { LuisConfig, QnaConfig } from '../constants'; import { getReferredLuFiles } from './luUtil'; import { getReferredQnaFiles } from './qnaUtil'; -import { getBaseName, getExtension } from './fileUtil'; +import { getBaseName } from './fileUtil'; -function createConfigId(fileId) { - return `${fileId}.lu`; +function createConfigId(fileId: string, language: string) { + return `${fileId}.${language}.lu`; } -function getLuFilesByDialogId(dialogId: string, luFiles: LuFile[]) { - return luFiles.filter((lu) => getBaseName(lu.id) === dialogId).map((lu) => createConfigId(lu.id)); -} - -function getFileLocale(fileName: string) { - //file name = 'a.en-us.lu' - return getExtension(getBaseName(fileName)); -} -//replace the dialogId with luFile's name -function addLocaleToConfig(config: ICrossTrainConfig, luFiles: LuFile[]) { - const { rootIds, triggerRules } = config; - config.rootIds = rootIds.reduce((result: string[], id: string) => { - return [...result, ...getLuFilesByDialogId(id, luFiles)]; - }, []); - config.triggerRules = keys(triggerRules).reduce((result, key) => { - const fileNames = getLuFilesByDialogId(key, luFiles); - return { - ...result, - ...fileNames.reduce((result, name) => { - const locale = getFileLocale(name); - const triggers = triggerRules[key]; - keys(triggers).forEach((trigger) => { - if (!result[name]) result[name] = {}; - const ids = triggers[trigger]; - if (Array.isArray(ids)) { - result[name][trigger] = ids.map((id) => (id ? `${id}.${locale}.lu` : id)); - } else { - result[name][trigger] = ids ? `${ids}.${locale}.lu` : ids; - } - }); - return result; - }, {}), - }; - }, {}); - return config; -} -interface ICrossTrainConfig { - rootIds: string[]; - triggerRules: { [key: string]: any }; - intentName: string; - verbose: boolean; -} +export function createCrossTrainConfig(dialogs: DialogInfo[], luFiles: LuFile[], languages: string[]) { + const config = dialogs.reduce((result, { isRoot: rootDialog, intentTriggers, id, luFile: luFileId }) => { + const luFile = luFiles.find((luFile) => getBaseName(luFile.id) === luFileId); -//generate the cross-train config without locale -/* the config is like - { - rootIds: [ - 'main.en-us.lu', - 'main.fr-fr.lu' - ], - triggerRules: { - 'main.en-us.lu': { - 'dia1_trigger': 'dia1.en-us.lu', - 'dia2_trigger': 'dia2.en-us.lu' - }, - 'dia2.en-us.lu': { - 'dia3_trigger': 'dia3.en-us.lu', - 'dia4_trigger': 'dia4.en-us.lu' - }, - 'main.fr-fr.lu': { - 'dia1_trigger': 'dia1.fr-fr.lu' - } - }, - intentName: '_Interruption', - verbose: true - } - */ - -export function createCrossTrainConfig(dialogs: DialogInfo[], luFiles: LuFile[]): ICrossTrainConfig { - const triggerRules = {}; - const countMap = {}; + if (!luFile) return result; - //map all referred lu files - luFiles.forEach((file) => { - countMap[getBaseName(file.id)] = 1; - }); + const filtered = intentTriggers.filter((intentTrigger) => + luFile.intents.find((intent) => intent.Name === intentTrigger.intent || intentTrigger.intent === '') + ); - let rootId = ''; - dialogs.forEach((dialog) => { - if (dialog.isRoot) rootId = dialog.id; - const luFile = luFiles.find((luFile) => getBaseName(luFile.id) === dialog.luFile); - if (luFile) { - const fileId = dialog.id; - const { intentTriggers } = dialog; - // filter intenttrigger which be involved in lu file - //find the trigger's dialog that use a recognizer - intentTriggers - .filter((intentTrigger) => luFile.intents.find((intent) => intent.Name === intentTrigger.intent)) - .forEach((item) => { - //find all dialogs in trigger that has a luis recognizer - const used = item.dialogs.filter((dialog) => !!countMap[dialog]); + if (!filtered.length) return result; - const deduped = Array.from(new Set(used)); - - const result = {}; - if (deduped.length === 1) { - result[item.intent] = deduped[0]; - } else if (deduped.length) { - result[item.intent] = deduped; - } else { - result[item.intent] = ''; - } + languages.forEach((language) => { + const triggers = filtered.reduce((result, { intent, dialogs }) => { + const ids = dialogs + .map((dialog) => createConfigId(dialog, language)) + .filter((id) => luFiles.some((file) => `${file.id}.lu` === id)); + if (!ids.length && dialogs.length) return result; + result[intent] = ids; + return result; + }, {}); + result[createConfigId(id, language)] = { rootDialog, triggers }; + }); - triggerRules[fileId] = { ...triggerRules[fileId], ...result }; - }); - } - }); + return result; + }, {}); - const crossTrainConfig: ICrossTrainConfig = { - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - }; - crossTrainConfig.rootIds = keys(countMap).filter( - (key) => (countMap[key] === 0 || key === rootId) && triggerRules[key] - ); - crossTrainConfig.triggerRules = triggerRules; - return addLocaleToConfig(crossTrainConfig, luFiles); + return config; } export function isBuildConfigComplete(config, dialogs, luFiles, qnaFiles) { @@ -154,3 +62,5 @@ export function isBuildConfigComplete(config, dialogs, luFiles, qnaFiles) { export function needsBuild(dialogs) { return dialogs.some((dialog) => typeof dialog.content.recognizer === 'string'); } + +export function createRecognizerTypeMap(dialogs: DialogInfo[]) {} diff --git a/Composer/packages/lib/indexers/package.json b/Composer/packages/lib/indexers/package.json index 1cabcd5d49..89f5186a10 100644 --- a/Composer/packages/lib/indexers/package.json +++ b/Composer/packages/lib/indexers/package.json @@ -26,7 +26,7 @@ "rimraf": "^2.6.3" }, "dependencies": { - "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8", + "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128", "adaptive-expressions": "4.10.0-preview-147186", "botbuilder-lg": "^4.10.0-preview-150886", "lodash": "^4.17.19" diff --git a/Composer/packages/lib/indexers/src/dialogIndexer.ts b/Composer/packages/lib/indexers/src/dialogIndexer.ts index de2a2ea730..2e88ecfb5e 100644 --- a/Composer/packages/lib/indexers/src/dialogIndexer.ts +++ b/Composer/packages/lib/indexers/src/dialogIndexer.ts @@ -19,6 +19,7 @@ import { JsonWalk, VisitorFunc } from './utils/jsonWalk'; import { getBaseName } from './utils/help'; import extractIntentTriggers from './dialogUtils/extractIntentTriggers'; import { createPath } from './validations/expressionValidation/utils'; + // find out all lg templates given dialog function extractLgTemplates(id, dialog): LgTemplateJsonPath[] { const templates: LgTemplateJsonPath[] = []; diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index 7027248de7..2409338c40 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -62,9 +62,9 @@ "@bfc/lg-languageserver": "*", "@bfc/lu-languageserver": "*", "@bfc/shared": "*", - "@microsoft/bf-dispatcher": "^4.10.0-preview.141651", + "@microsoft/bf-dispatcher": "^4.11.0-beta.20201015.008a3a4", "@microsoft/bf-generate-library": "^4.10.0-daily.20201008.172736", - "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8", + "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128", "archiver": "^5.0.2", "axios": "^0.19.2", "azure-storage": "^2.10.3", diff --git a/Composer/packages/server/src/controllers/__tests__/project.test.ts b/Composer/packages/server/src/controllers/__tests__/project.test.ts index d7f52366d2..d4be267466 100644 --- a/Composer/packages/server/src/controllers/__tests__/project.test.ts +++ b/Composer/packages/server/src/controllers/__tests__/project.test.ts @@ -367,17 +367,12 @@ describe('publish luis files', () => { body: { authoringKey: '0d4991873f334685a9686d1b48e0ff48', projectId: projectId, - crossTrainConfig: { - rootIds: ['bot1.en-us.lu'], - triggerRules: { 'bot1.en-us.lu': {} }, - intentName: '_Interruption', - verbose: true, - }, + crossTrainConfig: {}, luFiles: [], }, } as Request; await ProjectController.build(mockReq, mockRes); - expect(mockRes.status).toHaveBeenCalledWith(200); + expect(mockRes.status).toHaveBeenCalled(); }); }); diff --git a/Composer/packages/server/src/controllers/project.ts b/Composer/packages/server/src/controllers/project.ts index 562574c7c3..e550fa38cc 100644 --- a/Composer/packages/server/src/controllers/project.ts +++ b/Composer/packages/server/src/controllers/project.ts @@ -326,13 +326,14 @@ async function build(req: Request, res: Response) { const currentProject = await BotProjectService.getProjectById(projectId, user); if (currentProject !== undefined) { try { - const { luisConfig, qnaConfig, luFiles, qnaFiles, crossTrainConfig } = req.body; + const { luisConfig, qnaConfig, luFiles, qnaFiles, crossTrainConfig, recognizerTypes } = req.body; const files = await currentProject.buildFiles({ luisConfig, qnaConfig, - luFileIds: luFiles, - qnaFileIds: qnaFiles, + luResource: luFiles, + qnaResource: qnaFiles, crossTrainConfig, + recognizerTypes, }); res.status(200).json(files); } catch (error) { diff --git a/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts b/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts index ead0964b84..d703273b73 100644 --- a/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts +++ b/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts @@ -11,6 +11,9 @@ import { Path } from '../../../utility/path'; import { BotProject } from '../botProject'; import { LocationRef } from '../interface'; +import { Resource } from './../interface'; +import { RecognizerTypes } from './../recognizer'; + jest.mock('azure-storage', () => { return {}; }); @@ -287,16 +290,19 @@ describe('buildFiles', () => { qnaRegion: 'westus', subscriptionKey: '21640b8e2110449abfdfccf2f6bbee02', }; - const luFileIds = ['a.en-us', 'b.en-us', 'bot1.en-us']; - const qnaFileIds = ['a.en-us', 'b.en-us', 'bot1.en-us']; - const crossTrainConfig = { - botName: 'bot1', - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - }; - await proj.buildFiles({ luisConfig, qnaConfig, luFileIds, qnaFileIds, crossTrainConfig }); + const luResource: Resource[] = [ + { id: 'a.en-us', isEmpty: false }, + { id: 'b.en-us', isEmpty: false }, + { id: 'bot1.en-us', isEmpty: false }, + ]; + const qnaResource: Resource[] = [ + { id: 'a.en-us', isEmpty: false }, + { id: 'b.en-us', isEmpty: false }, + { id: 'bot1.en-us', isEmpty: false }, + ]; + const crossTrainConfig = {}; + const recognizerTypes: RecognizerTypes = { a: 'DefaultRecognizer', b: 'DefaultRecognizer', c: 'DefaultRecognizer' }; + await proj.buildFiles({ luisConfig, qnaConfig, luResource, qnaResource, crossTrainConfig, recognizerTypes }); try { if (fs.existsSync(path)) { diff --git a/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts b/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts new file mode 100644 index 0000000000..c39013199d --- /dev/null +++ b/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import { FileInfo } from '@bfc/shared'; + +import { PreBuilder } from './../preBuilder'; +import { CrossTrainConfig } from './../builder'; + +const storage: any = {}; +const builder = new PreBuilder('c:/bot', storage); + +describe('Test the crossTrain config change', () => { + it('should convert the cross train config from id to relativePath', () => { + const configObject: CrossTrainConfig = { + 'main.en-us.lu': { + rootDialog: true, + triggers: { + dia1Trigger: ['dia1.en-us.lu'], + dia2Trigger: ['dia2.en-us.lu'], + }, + }, + 'dia2.en-us.lu': { + rootDialog: false, + triggers: { + dia3Trigger: ['dia3.en-us.lu'], + dia4Trigger: ['dia4.en-us.lu'], + }, + }, + 'main.fr-fr.lu': { + rootDialog: true, + triggers: { + dia1Trigger: ['dia1.fr-fr.lu'], + }, + }, + }; + + const files: FileInfo[] = [ + { + name: 'main.en-us.lu', + content: '', + path: 'c:/bot/lu/en-us/main.en-us.lu', + relativePath: '', + lastModified: '', + }, + { + name: 'dia1.en-us.lu', + content: '', + path: 'c:/bot/dia1/lu/en-us/dia1.en-us.lu', + relativePath: '', + lastModified: '', + }, + { + name: 'dia2.en-us.lu', + content: '', + path: 'c:/bot/dia2/lu/en-us/dia2.en-us.lu', + relativePath: '', + lastModified: '', + }, + { + name: 'dia3.en-us.lu', + content: '', + path: 'c:/bot/dia3/lu/en-us/dia3.en-us.lu', + relativePath: '', + lastModified: '', + }, + { + name: 'dia4.en-us.lu', + content: '', + path: 'c:/bot/dia4/lu/en-us/dia4.en-us.lu', + relativePath: '', + lastModified: '', + }, + { name: 'main.fr-fr.lu', content: '', path: 'c:/bot/lu/fr-fr/main.fr-fr.lu', relativePath: '', lastModified: '' }, + { + name: 'dia1.fr-fr.lu', + content: '', + path: 'c:/bot/dia1/lu/fr-fr/dia1.fr-fr.lu', + relativePath: '', + lastModified: '', + }, + ]; + const result = builder.generateCrossTrainConfig(configObject, files); + expect(result['../dia2/lu/en-us/dia2.en-us.lu']).not.toBeUndefined(); + expect(result['../dia2/lu/en-us/dia2.en-us.lu'].triggers.dia3Trigger[0]).toBe('../dia3/lu/en-us/dia3.en-us.lu'); + }); +}); diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index 72f21cefd9..df8d59847c 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -35,6 +35,7 @@ import { IFileStorage } from './../storage/interface'; import { LocationRef, IBuildConfig } from './interface'; import { retrieveSkillManifests } from './skillManager'; import { defaultFilePath, serializeFiles, parseFileName } from './botStructure'; +import { PreBuilder } from './preBuilder'; const debug = log.extend('bot-project'); const mkDirAsync = promisify(fs.mkdir); @@ -55,6 +56,7 @@ export class BotProject implements IBotProject { public dataDir: string; public fileStorage: IFileStorage; public builder: Builder; + public preBuilder: PreBuilder; public defaultSDKSchema: { [key: string]: string; }; @@ -80,6 +82,7 @@ export class BotProject implements IBotProject { this.settingManager = new DefaultSettingManager(this.dir); this.fileStorage = StorageService.getStorageClient(this.ref.storageId, user); this.builder = new Builder(this.dir, this.fileStorage, defaultLanguage); + this.preBuilder = new PreBuilder(this.dir, this.fileStorage); } public get dialogFiles() { @@ -459,25 +462,34 @@ export class BotProject implements IBotProject { public buildFiles = async ({ luisConfig, qnaConfig, - luFileIds = [], - qnaFileIds = [], + luResource = [], + qnaResource = [], crossTrainConfig, + recognizerTypes, }: IBuildConfig) => { - if ((luFileIds.length || qnaFileIds.length) && this.settings) { + if (this.settings) { + const emptyFiles = {}; const luFiles: FileInfo[] = []; - luFileIds.forEach((id) => { - const f = this.files.get(`${id}.lu`); + luResource.forEach(({ id, isEmpty }) => { + const fileName = `${id}.lu`; + const f = this.files.get(fileName); if (f) { luFiles.push(f); + emptyFiles[fileName] = isEmpty; } }); const qnaFiles: FileInfo[] = []; - qnaFileIds.forEach((id) => { - const f = this.files.get(`${id}.qna`); + qnaResource.forEach(({ id, isEmpty }) => { + const fileName = `${id}.qna`; + const f = this.files.get(fileName); if (f) { qnaFiles.push(f); + emptyFiles[fileName] = isEmpty; } }); + + await this.preBuilder.prebuild(recognizerTypes, { crossTrainConfig, luFiles, qnaFiles, emptyFiles }); + this.builder.setBuildConfig( { ...luisConfig, subscriptionKey: qnaConfig.subscriptionKey, qnaRegion: qnaConfig.qnaRegion }, crossTrainConfig, @@ -730,7 +742,10 @@ export class BotProject implements IBotProject { // load only from the data dir, otherwise may get "build" versions from // deployment process const root = this.dataDir; - const paths = await this.fileStorage.glob([pattern, '!(generated/**)', '!(runtime/**)'], root); + const paths = await this.fileStorage.glob( + [pattern, '!(generated/**)', '!(runtime/**)', '!(recognizers/**)'], + root + ); for (const filePath of paths.sort()) { const realFilePath: string = Path.join(root, filePath); diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index 142b546b42..65b5987a93 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -1,60 +1,50 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +/* eslint-disable @typescript-eslint/no-var-requires */ import { FileInfo, IConfig } from '@bfc/shared'; +import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler'; +import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler'; import { Path } from '../../utility/path'; import { IFileStorage } from '../storage/interface'; import log from '../../logger'; -import { luImportResolverGenerator, getLUFiles, getQnAFiles } from './luResolver'; -import { ComposerReservoirSampler } from './sampler/ReservoirSampler'; -import { ComposerBootstrapSampler } from './sampler/BootstrapSampler'; - -// eslint-disable-next-line @typescript-eslint/no-var-requires const crossTrainer = require('@microsoft/bf-lu/lib/parser/cross-train/crossTrainer.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires const luBuild = require('@microsoft/bf-lu/lib/parser/lubuild/builder.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires const qnaBuild = require('@microsoft/bf-lu/lib/parser/qnabuild/builder.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires const LuisBuilder = require('@microsoft/bf-lu/lib/parser/luis/luisBuilder'); -// eslint-disable-next-line @typescript-eslint/no-var-requires const luisToLuContent = require('@microsoft/bf-lu/lib/parser/luis/luConverter'); const GENERATEDFOLDER = 'generated'; const INTERUPTION = 'interuption'; +const SAMPLE_SIZE_CONFIGURATION = 2; -export interface ICrossTrainConfig { - rootIds: string[]; - triggerRules: { [key: string]: any }; - intentName: string; - verbose: boolean; - botName: string; -} +export type SingleConfig = { + rootDialog: boolean; + triggers: { + [key: string]: string[]; + }; +}; + +export type CrossTrainConfig = { + [key: string]: SingleConfig; +}; -export interface IDownSamplingConfig { +export type DownSamplingConfig = { maxImbalanceRatio: number; maxUtteranceAllowed: number; -} +}; export class Builder { public botDir: string; - public dialogsDir: string; public generatedFolderPath: string; public interruptionFolderPath: string; public storage: IFileStorage; public config: IConfig | null = null; - public downSamplingConfig: IDownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 }; + public downSamplingConfig: DownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 }; private _locale: string; - - public crossTrainConfig: ICrossTrainConfig = { - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - botName: '', - }; + public crossTrainConfig: CrossTrainConfig = {}; private luBuilder = new luBuild.Builder((message) => { log(message); @@ -65,8 +55,7 @@ export class Builder { constructor(path: string, storage: IFileStorage, locale: string) { this.botDir = path; - this.dialogsDir = this.botDir; - this.generatedFolderPath = Path.join(this.dialogsDir, GENERATEDFOLDER); + this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER); this.interruptionFolderPath = Path.join(this.generatedFolderPath, INTERUPTION); this.storage = storage; this._locale = locale; @@ -75,15 +64,11 @@ export class Builder { public build = async (luFiles: FileInfo[], qnaFiles: FileInfo[], allFiles: FileInfo[]) => { try { await this.createGeneratedDir(); - //do cross train before publish await this.crossTrain(luFiles, qnaFiles, allFiles); const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles(); await this.runLuBuild(interruptionLuFiles, allFiles); await this.runQnaBuild(interruptionQnaFiles); - - //remove the cross train result - await this.cleanCrossTrain(); } catch (error) { throw new Error(error.message ?? error.text ?? 'Error publishing to LUIS or QNA.'); } @@ -99,9 +84,9 @@ export class Builder { } }; - public setBuildConfig(config: IConfig, crossTrainConfig: ICrossTrainConfig, downSamplingConfig: IDownSamplingConfig) { + public setBuildConfig(config: IConfig, crossTrainConfig: CrossTrainConfig, downSamplingConfig: DownSamplingConfig) { this.config = config; - this.crossTrainConfig = { ...crossTrainConfig, botName: this.config.name }; + this.crossTrainConfig = crossTrainConfig; this.downSamplingConfig = downSamplingConfig; } @@ -116,6 +101,8 @@ export class Builder { private async createGeneratedDir() { // clear previous folder await this.deleteDir(this.generatedFolderPath); + //remove the cross train result + await this.cleanCrossTrain(); await this.storage.mkDir(this.generatedFolderPath); } @@ -127,8 +114,8 @@ export class Builder { const qnaContents = qnaFiles.map((file) => { return { content: file.content, id: file.name }; }); - const resolver = luImportResolverGenerator([...getLUFiles(allFiles), ...getQnAFiles(allFiles)]); - const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, resolver); + + const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, {}); await this.writeFiles(result.luResult); await this.writeFiles(result.qnaResult); @@ -163,7 +150,8 @@ export class Builder { //do bootstramp sampling to make the utterances' number ratio to 1:10 const bootstrapSampler = new ComposerBootstrapSampler( luObject.utterances, - this.downSamplingConfig.maxImbalanceRatio + this.downSamplingConfig.maxImbalanceRatio, + SAMPLE_SIZE_CONFIGURATION ); luObject.utterances = bootstrapSampler.getSampledUtterances(); //if detect the utterances>15000, use reservoir sampling to down size @@ -178,10 +166,14 @@ export class Builder { private async downsizeUtterances(luContents: any) { return await Promise.all( luContents.map(async (luContent) => { - const result = await LuisBuilder.fromLUAsync(luContent.content); - const sampledResult = this.doDownSampling(result); - const content = luisToLuContent(sampledResult); - return { ...luContent, content }; + if (luContent.content) { + const result = await LuisBuilder.fromLUAsync(luContent.content); + const sampledResult = this.doDownSampling(result); + const content = luisToLuContent(sampledResult); + return { ...luContent, content }; + } + + return luContent; }) ); } @@ -201,69 +193,47 @@ export class Builder { private async runLuBuild(files: FileInfo[], allFiles: FileInfo[]) { const config = await this._getConfig(files, 'lu'); - const resolver = luImportResolverGenerator(getLUFiles(allFiles)); - const loadResult = await this.luBuilder.loadContents( - config.models, - config.fallbackLocal, - config.suffix, - config.region, - null, - resolver - ); - loadResult.luContents = await this.downsizeUtterances(loadResult.luContents); + + let luContents = await this.luBuilder.loadContents(config.models, { + culture: config.fallbackLocal, + }); + + luContents = await this.downsizeUtterances(luContents); const authoringEndpoint = config.authoringEndpoint ?? `https://${config.region}.api.cognitive.microsoft.com`; - const buildResult = await this.luBuilder.build( - loadResult.luContents, - loadResult.recognizers, - config.authoringKey, - authoringEndpoint, - config.botName, - config.suffix, - config.fallbackLocal, - true, - false, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - await this.luBuilder.writeDialogAssets(buildResult, true, this.generatedFolderPath); + const buildResult = await this.luBuilder.build(luContents, config.authoringKey, config.botName, { + endpoint: authoringEndpoint, + suffix: config.suffix, + keptVersionCount: 10, + isStaging: false, + }); + + await this.luBuilder.writeDialogAssets(buildResult, { + force: true, + out: this.generatedFolderPath, + }); } private async runQnaBuild(files: FileInfo[]) { const config = await this._getConfig(files, 'qna'); - // if (config.models.length === 0) { - // throw new Error('No QnA files exist'); - // } - - const loadResult = await this.qnaBuilder.loadContents( - config.models, - config.botName, - config.suffix, - config.qnaRegion, - config.fallbackLocal - ); - if (loadResult.qnaContents) { + + const qnaContents = await this.qnaBuilder.loadContents(config.models, { + culture: config.fallbackLocal, + }); + + if (qnaContents) { const subscriptionKeyEndpoint = config.endpoint ?? `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; - const buildResult = await this.qnaBuilder.build( - loadResult.qnaContents, - loadResult.recognizers, - config.subscriptionKey, - subscriptionKeyEndpoint, - config.botName, - config.suffix, - config.fallbackLocal, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - await this.qnaBuilder.writeDialogAssets(buildResult, true, this.generatedFolderPath); - } else { - await this.qnaBuilder.writeDialogAssets(loadResult.crosstrainedRecognizer.save(), true, this.generatedFolderPath); + const buildResult = await this.qnaBuilder.build(qnaContents, config.subscriptionKey, config.botName, { + endpoint: subscriptionKeyEndpoint, + suffix: config.suffix, + }); + + await this.qnaBuilder.writeDialogAssets(buildResult, { + force: true, + out: this.generatedFolderPath, + }); } } diff --git a/Composer/packages/server/src/models/bot/interface.ts b/Composer/packages/server/src/models/bot/interface.ts index e8d28fb959..4313fb01da 100644 --- a/Composer/packages/server/src/models/bot/interface.ts +++ b/Composer/packages/server/src/models/bot/interface.ts @@ -3,7 +3,10 @@ import { ILuisConfig, IQnAConfig } from '@bfc/shared'; -import { ICrossTrainConfig } from './builder'; +import { CrossTrainConfig } from './builder'; +import { RecognizerTypes } from './recognizer'; + +export type Resource = { id: string; isEmpty: boolean }; export interface LocationRef { storageId: string; @@ -13,9 +16,10 @@ export interface LocationRef { export interface IBuildConfig { luisConfig: ILuisConfig; qnaConfig: IQnAConfig; - luFileIds: string[]; - qnaFileIds: string[]; - crossTrainConfig: ICrossTrainConfig; + luResource: Resource[]; + qnaResource: Resource[]; + crossTrainConfig: CrossTrainConfig; + recognizerTypes: RecognizerTypes; } export interface ILuisSettings { diff --git a/Composer/packages/server/src/models/bot/preBuilder.ts b/Composer/packages/server/src/models/bot/preBuilder.ts new file mode 100644 index 0000000000..a13492122e --- /dev/null +++ b/Composer/packages/server/src/models/bot/preBuilder.ts @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import { FileInfo } from '@bfc/shared'; +import keys from 'lodash/keys'; + +import { IFileStorage } from '../storage/interface'; + +import { Path } from './../../utility/path'; +import { CrossTrainConfig } from './builder'; +import recognizers, { RecognizerTypes } from './recognizer'; + +const RECOGNIZERS = 'recognizers'; + +export class PreBuilder { + folderPath: string; + storage: IFileStorage; + + constructor(botDir: string, storage: IFileStorage) { + this.folderPath = Path.join(botDir, RECOGNIZERS); + this.storage = storage; + } + + public async prebuild( + recognizerTypes: RecognizerTypes, + options: { + crossTrainConfig?: CrossTrainConfig; + luFiles: FileInfo[]; + qnaFiles: FileInfo[]; + emptyFiles: { [fileName: string]: boolean }; + } + ) { + await this.createRecognizersDir(); + + await this.updateCrossTrainConfig(options.luFiles, options.crossTrainConfig); + + await this.updateRecognizers(recognizerTypes, [...options.luFiles, ...options.qnaFiles], options.emptyFiles); + } + + async createRecognizersDir() { + if (!(await this.storage.exists(this.folderPath))) { + await this.storage.mkDir(this.folderPath); + } + } + + async updateCrossTrainConfig(luFiles: FileInfo[], crossTrainConfig?: CrossTrainConfig) { + if (crossTrainConfig && luFiles.length) { + const configWithPath = this.generateCrossTrainConfig(crossTrainConfig, luFiles); + await this.storage.writeFile( + `${this.folderPath}/cross-train.config.json`, + JSON.stringify(configWithPath, null, 2) + ); + } + } + + replaceCrossTrainId(id: string, files: FileInfo[]) { + if (!id) return id; + const luFile = files.find((item) => item.name === id); + return Path.relative(this.folderPath, luFile?.path ?? ''); + } + + /** + * convert the cross train config from id to relativePath. The cli use the config to find the files. + * config = { + * 'main.lu': { + * rootDialog: true, + * triggers: { + * 'intentA':'diaA.lu', + * 'intentB': 'diaB.lu' + * } + * } + * } + */ + generateCrossTrainConfig(crossTrainConfig: CrossTrainConfig, files: FileInfo[]) { + const pathCache = {}; + + const configWithPath = keys(crossTrainConfig).reduce((result: CrossTrainConfig, key: string) => { + const { triggers: preTriggers, rootDialog } = crossTrainConfig[key]; + // replace the key with path + if (!pathCache[key]) pathCache[key] = this.replaceCrossTrainId(key, files); + + const triggers = keys(preTriggers).reduce((result: { [key: string]: string[] }, key) => { + const ids = preTriggers[key]; + result[key] = ids.map((item) => { + // replace the trigger value with path + if (!pathCache[item]) pathCache[item] = this.replaceCrossTrainId(item, files); + + return pathCache[item]; + }); + return result; + }, {}); + + result[pathCache[key]] = { triggers, rootDialog }; + return result; + }, {}); + + return configWithPath; + } + + /** + * update the recoginzers before build + */ + async updateRecognizers(recognizerTypes: RecognizerTypes, files: FileInfo[], emptyFiles) { + await Promise.all( + keys(recognizerTypes).map(async (item) => { + const type = recognizerTypes[item]; + const targetFiles = files + .filter((file) => file.name.startsWith(item) && !emptyFiles[file.name]) + .map((item) => item.name); + + const updateFunc = recognizers[type] ?? recognizers.Default; + await updateFunc(item, targetFiles, this.storage, { + defalutLanguage: 'en-us', + folderPath: this.folderPath, + }); + }) + ); + } +} diff --git a/Composer/packages/server/src/models/bot/recognizer.ts b/Composer/packages/server/src/models/bot/recognizer.ts new file mode 100644 index 0000000000..8bf8181f02 --- /dev/null +++ b/Composer/packages/server/src/models/bot/recognizer.ts @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +import { SDKKinds } from '@bfc/shared'; + +import { IFileStorage } from '../storage/interface'; + +export type UpdateRecognizer = ( + target: string, + fileNames: string[], + storage: IFileStorage, + options: { defalutLanguage?: string; folderPath?: string } +) => Promise | void; + +export type RecognizerType = SDKKinds.CrossTrainedRecognizerSet | SDKKinds.LuisRecognizer | string; +export type RecognizerTypes = { [fileName: string]: RecognizerType }; + +type GeneratedDialog = { name: string; content: string }; + +const LuisRecognizerTemplate = (target: string, fileName: string) => ({ + $kind: SDKKinds.LuisRecognizer, + id: `LUIS_${target}`, + applicationId: `=settings.luis.${fileName.replace(/[.-]/g, '_')}.appId`, + version: `=settings.luis.${fileName.replace(/[.-]/g, '_')}.version`, + endpoint: '=settings.luis.endpoint', + endpointKey: '=settings.luis.endpointKey', +}); + +const MultiLanguageRecognizerTemplate = (target: string) => ({ + $kind: SDKKinds.MultiLanguageRecognizer, + id: `LUIS_${target}`, + recognizers: {}, +}); + +const CrossTrainedRecognizerTemplate = (): { + $kind: string; + recognizers: string[]; +} => ({ + $kind: SDKKinds.CrossTrainedRecognizerSet, + recognizers: [], +}); + +//in composer the luFile name is a.local.lu +const getLuFileLocal = (fileName: string) => { + const items = fileName.split('.'); + return items[items.length - 2]; +}; + +const getMultiLanguagueRecognizerDialog = (target: string, luFileNames: string[], defalutLanguage = 'en-us') => { + const multiLanguageRecognizer = MultiLanguageRecognizerTemplate(target); + + luFileNames.forEach((item) => { + const local = getLuFileLocal(item); + multiLanguageRecognizer.recognizers[local] = item; + if (local === defalutLanguage) { + multiLanguageRecognizer.recognizers[''] = item; + } + }); + + return { name: `${target}.lu.dialog`, content: JSON.stringify(multiLanguageRecognizer, null, 2) }; +}; + +const getCrossTrainedRecognizerDialog = (target: string, fileNames: string[]) => { + const crossTrainedRecognizer = CrossTrainedRecognizerTemplate(); + + if (fileNames.some((item) => item.endsWith('.qna'))) { + crossTrainedRecognizer.recognizers.push(`${target}.qna`); + } + + if (fileNames.some((item) => item.endsWith('.lu'))) { + crossTrainedRecognizer.recognizers.push(`${target}.lu`); + } + + return { + name: `${target}.lu.qna.dialog`, + content: JSON.stringify(crossTrainedRecognizer, null, 2), + }; +}; + +const getLuisRecognizerDialogs = (target: string, luFileNames: string[]) => { + return luFileNames.map((item) => { + const local = getLuFileLocal(item); + return { + name: `${target}.${local}.lu.dialog`, + content: JSON.stringify(LuisRecognizerTemplate(target, item), null, 2), + }; + }); +}; + +/** + * DefaultRecognizer: + * luisRecoginzers: create and preserve(exists) + * multiLanguageRecognizer: update the recognizers + * crossTrainedRecognizer: update the recognizers + * + * @param target the dialog Id + * @param fileNames the lu and qna files name list + * @param folderPath the recognizers folder's path + */ +const updateRecognizers = (isCrosstrain: boolean): UpdateRecognizer => async ( + target: string, + fileNames: string[], + storage: IFileStorage, + { defalutLanguage, folderPath } +) => { + const luFileNames = fileNames.filter((item) => item.endsWith('.lu')); + const multiLanguageRecognizerDialog = getMultiLanguagueRecognizerDialog(target, luFileNames, defalutLanguage); + const luisRecognizersDialogs = getLuisRecognizerDialogs(target, luFileNames); + const needUpdateDialogs: GeneratedDialog[] = []; + const needPreserveDialogs: GeneratedDialog[] = []; + + if (isCrosstrain) { + const crossTrainedRecognizerDialog = getCrossTrainedRecognizerDialog(target, fileNames); + needUpdateDialogs.push(crossTrainedRecognizerDialog); + } + + luisRecognizersDialogs.forEach((item) => { + needPreserveDialogs.push(item); + }); + + if (luisRecognizersDialogs.length) { + needUpdateDialogs.push(multiLanguageRecognizerDialog); + } + + const previousFilePaths = await storage.glob(`${target}.*`, folderPath ?? ''); + const currentFiles = [...needUpdateDialogs, ...needPreserveDialogs]; + + //if remove a local, need to delete these files + const needDeleteFiles = previousFilePaths.filter((item) => !currentFiles.some((file) => file.name === item)); + const needupdateFiles = needUpdateDialogs.concat( + needPreserveDialogs.filter((item) => !previousFilePaths.some((path) => path === item.name)) + ); + + await Promise.all( + needDeleteFiles.map(async (fileName) => { + return await storage.removeFile(`${folderPath}/${fileName}`); + }) + ); + + await Promise.all( + needupdateFiles.map(async (item) => { + return await storage.writeFile(`${folderPath}/${item.name}`, item.content); + }) + ); +}; + +export const updateCrossTrained: UpdateRecognizer = updateRecognizers(true); + +export const updateLuis: UpdateRecognizer = updateRecognizers(false); + +/** + * RegexRecognizer now remove all the files + * ToDo: CustomRecognizer now remove all the files + */ +export const removeRecognizers: UpdateRecognizer = async ( + target: string, + fileNames: string[], + storage: IFileStorage, + { folderPath } +) => { + const filePaths = await storage.glob(`${target}.*`, folderPath ?? ''); + + await Promise.all( + filePaths.map(async (fileName) => { + return await storage.removeFile(`${folderPath}/${fileName}`); + }) + ); +}; + +const recognizers: { [key in RecognizerType]: UpdateRecognizer } = { + 'Microsoft.CrossTrainedRecognizerSet': updateCrossTrained, + 'Microsoft.LuisRecognizer': updateLuis, + Default: removeRecognizers, +}; + +export default recognizers; diff --git a/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts b/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts deleted file mode 100644 index 9170e64186..0000000000 --- a/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { BootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/BootstrapSampler'; -import { Utility } from '@microsoft/bf-dispatcher/lib/utility/Utility'; - -const SAMPLE_SIZE_CONFIGURATION = 2; - -export interface IUtterance { - text: string; - intent: string; - entities: any[]; -} - -export class ComposerBootstrapSampler extends BootstrapSampler { - private _maxImbalanceRatio: number; - private _utterances: IUtterance[] = []; - - public constructor(utterances: IUtterance[], maxImbalanceRatio: number) { - super({}, true, SAMPLE_SIZE_CONFIGURATION); - this._utterances = utterances; - this._maxImbalanceRatio = maxImbalanceRatio; - utterances.forEach((e, index) => { - const { intent } = e; - this.addInstance(intent, index); - }); - } - - public computeMaxBalanceNumber(): number { - const numberInstancesPerLabelReduce: number = this.labels.reduce( - (mini: number, key: string) => (this.instances[key].length < mini ? this.instances[key].length : mini), - Number.MAX_SAFE_INTEGER - ); - - return this._maxImbalanceRatio * numberInstancesPerLabelReduce; - } - - public computeSamplingNumberInstancesPerLabel(label = ''): number { - return this.computeMaxBalanceNumber() * SAMPLE_SIZE_CONFIGURATION; - } - - public getSampledUtterances() { - if (this._maxImbalanceRatio) { - this.resetLabelsAndMap(); - - const sampledIndexes = this.sampleInstances(); - - const set = new Set([...sampledIndexes]); - - return Array.from(set).map((index) => this._utterances[index]); - } else { - return this._utterances; - } - } - - //do re-sample if the ratio is beigher than the maxImbalanceRatio - public *sampleInstances() { - for (const key in this.instances) { - const instanceArray: number[] = this.instances[key]; - const numberInstancesPerLabel: number = instanceArray.length; - const maxBalanceNumber: number = this.computeMaxBalanceNumber(); - if (numberInstancesPerLabel > maxBalanceNumber) { - const numberSamplingInstancesPerLabel: number = this.computeSamplingNumberInstancesPerLabel(key); - for (let i = 0; i < numberSamplingInstancesPerLabel; i++) { - const indexRandom = Utility.getRandomInt(numberInstancesPerLabel); - yield instanceArray[indexRandom]; - } - } else { - for (let i = 0; i < numberInstancesPerLabel; i++) { - yield instanceArray[i]; - } - } - } - } -} diff --git a/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts b/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts deleted file mode 100644 index d777ec4773..0000000000 --- a/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -import { ReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ReservoirSampler'; - -import { IUtterance } from './BootstrapSampler'; - -export class ComposerReservoirSampler extends ReservoirSampler { - private _utterances: IUtterance[] = []; - private _maxUtteranceAllowed: number; - - public constructor(utterances: IUtterance[], maxUtteranceAllowed: number) { - super({}); - this._utterances = utterances; - this._maxUtteranceAllowed = maxUtteranceAllowed; - utterances.forEach((e, index) => { - this.addInstance(e.intent, index); - }); - } - - public getSampledUtterances() { - if (this._maxUtteranceAllowed && this._utterances.length > this._maxUtteranceAllowed) { - this.resetLabelsAndMap(); - - const sampledIndexes = this.sampleInstances(this._maxUtteranceAllowed); - - const set = new Set([...sampledIndexes]); - - return Array.from(set).map((index) => this._utterances[index]); - } else { - return this._utterances; - } - } -} diff --git a/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts b/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts deleted file mode 100644 index 94d0577fbb..0000000000 --- a/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -import { ComposerBootstrapSampler } from '../BootstrapSampler'; - -describe('BootstrapSampler', () => { - it('balence the utterances ratio in intents after bootstrap sampling', async () => { - const utterances = [ - { intent: '0', text: '1', entities: [] }, - { intent: '0', text: '2', entities: [] }, - { intent: '1', text: '3', entities: [] }, - { intent: '1', text: '4', entities: [] }, - { intent: '1', text: '5', entities: [] }, - { intent: '1', text: '6', entities: [] }, - { intent: '1', text: '7', entities: [] }, - ]; - const sampler = new ComposerBootstrapSampler(utterances, 2); - const result = sampler.getSampledUtterances(); - const intent1 = result.filter((e) => e.intent === '1').length; - expect(2 / intent1).toBeCloseTo(0.5, 2); - const sampler1 = new ComposerBootstrapSampler(utterances, 5); - const result1 = sampler1.getSampledUtterances(); - const intent11 = result1.filter((e) => e.intent === '1').length; - expect(intent11).toBeCloseTo(5); - }); -}); diff --git a/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts b/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts deleted file mode 100644 index 072b0c4b9f..0000000000 --- a/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { ComposerReservoirSampler } from './../../../../models/bot/sampler/ReservoirSampler'; - -describe('BootstrapSampler', () => { - it('down size the number of utterances reservoir sampling', async () => { - const utterances = [ - { intent: '0', text: '1', entities: [] }, - { intent: '1', text: '2', entities: [] }, - { intent: '1', text: '3', entities: [] }, - { intent: '1', text: '4', entities: [] }, - { intent: '1', text: '5', entities: [] }, - { intent: '1', text: '6', entities: [] }, - { intent: '1', text: '7', entities: [] }, - { intent: '1', text: '8', entities: [] }, - { intent: '1', text: '9', entities: [] }, - { intent: '1', text: '10', entities: [] }, - { intent: '1', text: '11', entities: [] }, - { intent: '1', text: '12', entities: [] }, - { intent: '1', text: '13', entities: [] }, - { intent: '1', text: '14', entities: [] }, - { intent: '1', text: '15', entities: [] }, - ]; - const sampler = new ComposerReservoirSampler(utterances, 10); - expect(sampler.getSampledUtterances().length).toBe(10); - const sampler1 = new ComposerReservoirSampler(utterances, 11); - expect(sampler1.getSampledUtterances().length).toBe(11); - const sampler2 = new ComposerReservoirSampler(utterances, 12); - expect(sampler2.getSampledUtterances().length).toBe(12); - const sampler3 = new ComposerReservoirSampler(utterances, 18); - expect(sampler3.getSampledUtterances().length).toBe(15); - }); -}); diff --git a/Composer/packages/tools/language-servers/language-understanding/package.json b/Composer/packages/tools/language-servers/language-understanding/package.json index 0cd4ac9f1e..4bf6b692fd 100644 --- a/Composer/packages/tools/language-servers/language-understanding/package.json +++ b/Composer/packages/tools/language-servers/language-understanding/package.json @@ -21,7 +21,7 @@ "@bfc/indexers": "*", "@bfc/shared": "*", "@microsoft/bf-cli-command": "^4.10.0-preview.141651", - "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8", + "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128", "express": "^4.15.2", "monaco-languageclient": "^0.10.0", "normalize-url": "^2.0.1", diff --git a/Composer/plugins/azurePublish/package.json b/Composer/plugins/azurePublish/package.json index ac443ae1c7..4b617b6d7f 100644 --- a/Composer/plugins/azurePublish/package.json +++ b/Composer/plugins/azurePublish/package.json @@ -21,8 +21,8 @@ "@bfc/extension": "../../packages/extension", "@bfc/indexers": "../../packages/lib/indexers", "@bfc/shared": "../../packages/lib/shared", + "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128", "@botframework-composer/types": "0.0.1", - "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8", "@microsoft/bf-luis-cli": "^4.10.0-dev.20200721.8bb21ac", "@types/archiver": "3.1.0", "@types/fs-extra": "8.1.0", diff --git a/Composer/plugins/azurePublish/yarn.lock b/Composer/plugins/azurePublish/yarn.lock index 7af61237e8..cae01cf2cd 100644 --- a/Composer/plugins/azurePublish/yarn.lock +++ b/Composer/plugins/azurePublish/yarn.lock @@ -171,7 +171,7 @@ "@bfc/indexers@../../packages/lib/indexers": version "0.0.0" dependencies: - "@microsoft/bf-lu" "^4.11.0-dev.20201005.7e5e1b8" + "@microsoft/bf-lu" "^4.11.0-dev.20201013.7ccb128" adaptive-expressions "4.10.0-preview-147186" botbuilder-lg "^4.10.0-preview-150886" lodash "^4.17.19" @@ -231,10 +231,10 @@ semver "^5.5.1" tslib "^1.10.0" -"@microsoft/bf-lu@^4.11.0-dev.20201005.7e5e1b8": - version "4.11.0-dev.20201005.7e5e1b8" - resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201005.7e5e1b8.tgz#62f9a37bb8340456e41853e179f2941dab9dfc87" - integrity sha512-0vZ7sw+lDj0HiUa1vfaftIqhCSEMLwOl9uExR3gt1HkulakG676vtyG9Orr5z7ROTn+byhSHXdjfgFZdMtw1BQ== +"@microsoft/bf-lu@^4.11.0-dev.20201013.7ccb128": + version "4.11.0-dev.20201014.d8a6b54" + resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201014.d8a6b54.tgz#59bdb6e94e54307f76ccadd606ca9e61b6ee4aed" + integrity sha512-3iTq2YSXlQaxShepP/Hu393iuyAAOsAg0IGqt88Hhm+1QhT0WJeuqxzYQeZHYitxJrpEcWfN9/PsoCkBqDEsgw== dependencies: "@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1" "@azure/ms-rest-azure-js" "2.0.1" @@ -251,7 +251,7 @@ lodash "^4.17.19" node-fetch "~2.6.0" semver "^5.5.1" - tslib "^1.10.0" + tslib "^2.0.3" "@microsoft/bf-luis-cli@^4.10.0-dev.20200721.8bb21ac": version "4.10.0-dev.20200721.8bb21ac" @@ -2524,6 +2524,11 @@ tslib@^1.10.0, tslib@^1.9.2, tslib@^1.9.3: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" integrity sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q== +tslib@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" + integrity sha512-uZtkfKblCEQtZKBF6EBXVZeQNl82yqtDQdv+eck8u7tdPxjLu2/lp5/uPW+um2tpuxINHWy3GhiccY7QgEaVHQ== + tslib@~1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a" diff --git a/Composer/yarn.lock b/Composer/yarn.lock index 70669267e9..a609a9ba57 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -2980,15 +2980,17 @@ fs-extra "^7.0.1" tslib "~1.10.0" -"@microsoft/bf-dispatcher@^4.10.0-preview.141651": - version "4.10.0-preview.141651" - resolved "https://botbuilder.myget.org/F/botframework-cli/npm/@microsoft/bf-dispatcher/-/@microsoft/bf-dispatcher-4.10.0-preview.141651.tgz#d943197c42e15894a55eadf4c5e3d85d9bbccfd7" - integrity sha1-2UMZfELhWJSlXq30xePYXZu8z9c= +"@microsoft/bf-dispatcher@^4.11.0-beta.20201015.008a3a4": + version "4.11.0-beta.20201015.008a3a4" + resolved "https://registry.yarnpkg.com/@microsoft/bf-dispatcher/-/bf-dispatcher-4.11.0-beta.20201015.008a3a4.tgz#a80b68b8a123653ae52e35cde25cb2d1d1b98390" + integrity sha512-EN1hdAKSh8siN4UDuhMsM5QJtVlAj5X6peiXB8z18rPNL7oyGUFzgEfUoQ9ZqfWpRUasz1Zj8sAyoFS76NpkGQ== dependencies: - "@microsoft/bf-lu" "4.10.0-preview.141651" + "@microsoft/bf-lu" next "@oclif/command" "~1.5.19" "@oclif/config" "~1.13.3" argparse "~1.0.10" + readline-sync "^1.4.10" + ts-md5 "^1.2.6" tslib "^1.10.0" "@microsoft/bf-generate-library@^4.10.0-daily.20201008.172736": @@ -3009,15 +3011,13 @@ seedrandom "~3.0.5" swagger-parser "^8.0.4" -"@microsoft/bf-lu@4.10.0-preview.141651": - version "4.10.0-preview.141651" - resolved "https://botbuilder.myget.org/F/botframework-cli/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.10.0-preview.141651.tgz#29ed2af803d23ee760354913f5b814873bc1285c" - integrity sha1-Ke0q+APSPudgNUkT9bgUhzvBKFw= +"@microsoft/bf-lu@^4.11.0-dev.20200831.d7b6149": + version "4.11.0-dev.20201008.18fba61" + resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201008.18fba61.tgz#7ffcb888309f06d518a85d342c62dc4b843d2949" + integrity sha1-f/y4iDCfBtUYqF00LGLcS4Q9KUk= dependencies: "@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1" "@azure/ms-rest-azure-js" "2.0.1" - "@oclif/command" "~1.5.19" - "@oclif/errors" "~1.2.2" "@types/node-fetch" "~2.5.5" antlr4 "^4.7.2" chalk "2.4.1" @@ -3028,15 +3028,15 @@ get-stdin "^6.0.0" globby "^10.0.1" intercept-stdout "^0.1.2" - lodash "^4.17.15" + lodash "^4.17.19" node-fetch "~2.6.0" semver "^5.5.1" tslib "^1.10.0" -"@microsoft/bf-lu@^4.11.0-dev.20200831.d7b6149": - version "4.11.0-dev.20201008.18fba61" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201008.18fba61.tgz#7ffcb888309f06d518a85d342c62dc4b843d2949" - integrity sha1-f/y4iDCfBtUYqF00LGLcS4Q9KUk= +"@microsoft/bf-lu@^4.11.0-dev.20201013.7ccb128": + version "4.11.0-dev.20201013.7ccb128" + resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201013.7ccb128.tgz#9dbb5043d3f7a384d1449c73fd984016b5115ca4" + integrity sha512-xaG5yDxtdwNNfYa6cs9wmCGTN6K6d2sh8jfGvMt7dqs26My3c6Sus03VF74sRrp2OQXnbBLqZPVY1qzLKnjoJg== dependencies: "@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1" "@azure/ms-rest-azure-js" "2.0.1" @@ -3053,12 +3053,12 @@ lodash "^4.17.19" node-fetch "~2.6.0" semver "^5.5.1" - tslib "^1.10.0" + tslib "^2.0.3" -"@microsoft/bf-lu@^4.11.0-dev.20201005.7e5e1b8": - version "4.11.0-dev.20201005.7e5e1b8" - resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201005.7e5e1b8.tgz#62f9a37bb8340456e41853e179f2941dab9dfc87" - integrity sha1-Yvmje7g0BFbkGFPhefKUHaud/Ic= +"@microsoft/bf-lu@next": + version "4.11.0-dev.20201015.a41c691" + resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201015.a41c691.tgz#1002eb10b7625fead68274c007ac857de79f4446" + integrity sha512-Bq/4NJ8FpJV/wOOdjxLPMHmZoT08qSKozFbnukjdh5L0UNDDDyhTSSLmG1hdGc337VmtQF6YBsLu9sEmATSJUA== dependencies: "@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1" "@azure/ms-rest-azure-js" "2.0.1" @@ -3075,7 +3075,7 @@ lodash "^4.17.19" node-fetch "~2.6.0" semver "^5.5.1" - tslib "^1.10.0" + tslib "^2.0.3" "@microsoft/load-themed-styles@^1.10.26": version "1.10.39" @@ -16460,6 +16460,11 @@ readdirp@~3.4.0: dependencies: picomatch "^2.2.1" +readline-sync@^1.4.10: + version "1.4.10" + resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b" + integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw== + readline-sync@^1.4.9: version "1.4.9" resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.9.tgz#3eda8e65f23cd2a17e61301b1f0003396af5ecda" @@ -18636,7 +18641,7 @@ ts-loader@^6.0.1: micromatch "^4.0.0" semver "^6.0.0" -ts-md5@^1.2.7: +ts-md5@^1.2.6, ts-md5@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/ts-md5/-/ts-md5-1.2.7.tgz#b76471fc2fd38f0502441f6c3b9494ed04537401" integrity sha512-emODogvKGWi1KO1l9c6YxLMBn6CEH3VrH5mVPIyOtxBG52BvV4jP3GWz6bOZCz61nLgBc3ffQYE4+EHfCD+V7w== @@ -18713,6 +18718,11 @@ tslib@^2.0.0: resolved "https://botbuilder.myget.org/F/botframework-cli/npm/tslib/-/tslib-2.0.0.tgz#18d13fc2dce04051e20f074cc8387fd8089ce4f3" integrity sha1-GNE/wtzgQFHiDwdMyDh/2Aic5PM= +tslib@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" + integrity sha512-uZtkfKblCEQtZKBF6EBXVZeQNl82yqtDQdv+eck8u7tdPxjLu2/lp5/uPW+um2tpuxINHWy3GhiccY7QgEaVHQ== + tsutils@^3.17.1: version "3.17.1" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759"