diff --git a/Composer/packages/client/__tests__/utils/buildUtil.test.ts b/Composer/packages/client/__tests__/utils/buildUtil.test.ts
index ce3e9fb67d..1a19bf9e16 100644
--- a/Composer/packages/client/__tests__/utils/buildUtil.test.ts
+++ b/Composer/packages/client/__tests__/utils/buildUtil.test.ts
@@ -80,14 +80,11 @@ describe('createCrossTrainConfig', () => {
{ id: 'dia5.en-us' },
{ id: 'dia6.en-us' },
];
- const config = createCrossTrainConfig(dialogs as DialogInfo[], luFiles as LuFile[]);
- expect(config.rootIds.length).toEqual(1);
- expect(config.rootIds[0]).toEqual('main.en-us.lu');
- expect(config.triggerRules['main.en-us.lu'].dia1_trigger).toEqual('dia1.en-us.lu');
- expect(config.triggerRules['main.en-us.lu'].no_dialog).toEqual('');
- expect(config.triggerRules['main.en-us.lu'].dia1_trigger).toEqual('dia1.en-us.lu');
- expect(config.triggerRules['main.en-us.lu'].dias_trigger.length).toBe(2);
- expect(config.triggerRules['dia1.en-us.lu'].dia3_trigger).toEqual('dia3.en-us.lu');
- expect(config.triggerRules['dia1.en-us.lu']['dia4.en-us.lu']).toBeUndefined();
+ const config = createCrossTrainConfig(dialogs as DialogInfo[], luFiles as LuFile[], ['en-us']);
+ expect(config['main.en-us.lu'].rootDialog).toBeTruthy();
+ expect(config['main.en-us.lu'].triggers.dia1_trigger[0]).toEqual('dia1.en-us.lu');
+ expect(config['main.en-us.lu'].triggers.no_dialog.length).toEqual(0);
+ expect(config['main.en-us.lu'].triggers.dia2_trigger[0]).toEqual('dia2.en-us.lu');
+ expect(config['main.en-us.lu'].triggers.dias_trigger.length).toBe(2);
});
});
diff --git a/Composer/packages/client/src/components/TestController/TestController.tsx b/Composer/packages/client/src/components/TestController/TestController.tsx
index 27549b7274..1bdd05e62d 100644
--- a/Composer/packages/client/src/components/TestController/TestController.tsx
+++ b/Composer/packages/client/src/components/TestController/TestController.tsx
@@ -9,6 +9,7 @@ import { PrimaryButton } from 'office-ui-fabric-react/lib/Button';
import formatMessage from 'format-message';
import { useRecoilValue } from 'recoil';
import { IConfig, IPublishConfig, defaultPublishConfig } from '@bfc/shared';
+import { useRecognizerConfig } from '@bfc/extension-client';
import {
botEndpointsState,
@@ -67,6 +68,7 @@ export const TestController: React.FC<{ projectId: string }> = (props) => {
const settings = useRecoilValue(settingsState(projectId));
const qnaFiles = useRecoilValue(qnaFilesState(projectId));
const botLoadErrorMsg = useRecoilValue(botLoadErrorState(projectId));
+ const { recognizers } = useRecognizerConfig();
const botEndpoints = useRecoilValue(botEndpointsState);
const {
@@ -156,12 +158,18 @@ export const TestController: React.FC<{ projectId: string }> = (props) => {
setBotStatus(BotStatus.publishing, projectId);
dismissDialog();
const { luis, qna } = config;
+ const recognizerTypes = dialogs.reduce((result, file) => {
+ const recognizer = recognizers.filter((r) => r.isSelected && r.isSelected(file.content.recognizer));
+ result[file.id] = recognizer[0]?.id || '';
+ return result;
+ }, {});
+
await setSettings(projectId, {
...settings,
luis: luis,
qna: Object.assign({}, settings.qna, qna),
});
- await build(luis, qna, projectId);
+ await build(luis, qna, recognizerTypes, projectId);
}
async function handleLoadBot() {
diff --git a/Composer/packages/client/src/components/Toolbar.tsx b/Composer/packages/client/src/components/Toolbar.tsx
index c50fde11f0..11ab4b1913 100644
--- a/Composer/packages/client/src/components/Toolbar.tsx
+++ b/Composer/packages/client/src/components/Toolbar.tsx
@@ -3,11 +3,17 @@
/** @jsx jsx */
import { jsx, css } from '@emotion/core';
-import { Fragment } from 'react';
+import { Fragment, useMemo } from 'react';
import formatMessage from 'format-message';
import { NeutralColors } from '@uifabric/fluent-theme';
import { ActionButton, CommandButton } from 'office-ui-fabric-react/lib/Button';
import { IContextualMenuProps, IIconProps } from 'office-ui-fabric-react/lib';
+import { EditorExtension, mergePluginConfigs, PluginConfig } from '@bfc/extension-client';
+import { useRecoilValue } from 'recoil';
+
+import plugins from '../plugins';
+import { currentProjectIdState, schemasState } from '../recoilModel';
+import { useShell } from '../shell/useShell';
// -------------------- Styles -------------------- //
@@ -98,6 +104,15 @@ type ToolbarProps = {
// fabric-ui IButtonProps interface}
export const Toolbar = (props: ToolbarProps) => {
const { toolbarItems = [], ...rest } = props;
+ const projectId = useRecoilValue(currentProjectIdState);
+ const schemas = useRecoilValue(schemasState(projectId));
+ const shellForPropertyEditor = useShell('DesignPage', projectId);
+
+ const pluginConfig: PluginConfig = useMemo(() => {
+ const sdkUISchema = schemas?.ui?.content ?? {};
+ const userUISchema = schemas?.uiOverrides?.content ?? {};
+ return mergePluginConfigs({ uiSchema: sdkUISchema }, plugins, { uiSchema: userUISchema });
+ }, [schemas?.ui?.content, schemas?.uiOverrides?.content]);
const left: IToolbarItem[] = [];
const right: IToolbarItem[] = [];
@@ -113,9 +128,11 @@ export const Toolbar = (props: ToolbarProps) => {
}
return (
-
-
{left.map(renderItemList)}
-
{right.map(renderItemList)}
-
+
+
+
{left.map(renderItemList)}
+
{right.map(renderItemList)}
+
+
);
};
diff --git a/Composer/packages/client/src/recoilModel/dispatchers/builder.ts b/Composer/packages/client/src/recoilModel/dispatchers/builder.ts
index 81c92cca33..c2af7d8136 100644
--- a/Composer/packages/client/src/recoilModel/dispatchers/builder.ts
+++ b/Composer/packages/client/src/recoilModel/dispatchers/builder.ts
@@ -12,6 +12,7 @@ import httpClient from '../../utils/httpUtil';
import luFileStatusStorage from '../../utils/luFileStatusStorage';
import qnaFileStatusStorage from '../../utils/qnaFileStatusStorage';
import { luFilesState, qnaFilesState, dialogsState, botStatusState, botLoadErrorState } from '../atoms';
+import { settingsState } from '../atoms/botState';
const checkEmptyQuestionOrAnswerInQnAFile = (sections) => {
return sections.some((s) => !s.Answer || s.Questions.some((q) => !q.content));
@@ -22,13 +23,14 @@ export const builderDispatcher = () => {
({ set, snapshot }: CallbackInterface) => async (
luisConfig: ILuisConfig,
qnaConfig: IQnAConfig,
+ recognizerTypes: { [fileName: string]: string },
projectId: string
) => {
const dialogs = await snapshot.getPromise(dialogsState(projectId));
const luFiles = await snapshot.getPromise(luFilesState(projectId));
const qnaFiles = await snapshot.getPromise(qnaFilesState(projectId));
+ const settings = await snapshot.getPromise(settingsState(projectId));
const referredLuFiles = luUtil.checkLuisBuild(luFiles, dialogs);
-
const errorMsg = qnaFiles.reduce(
(result, file) => {
if (
@@ -48,15 +50,15 @@ export const builderDispatcher = () => {
return;
}
try {
- //TODO crosstrain should add locale
- const crossTrainConfig = buildUtil.createCrossTrainConfig(dialogs, referredLuFiles);
+ const crossTrainConfig = buildUtil.createCrossTrainConfig(dialogs, referredLuFiles, settings.languages);
await httpClient.post(`/projects/${projectId}/build`, {
luisConfig,
qnaConfig,
projectId,
crossTrainConfig,
- luFiles: referredLuFiles.map((file) => file.id),
- qnaFiles: qnaFiles.map((file) => file.id),
+ recognizerTypes,
+ luFiles: referredLuFiles.map((file) => ({ id: file.id, isEmpty: file.empty })),
+ qnaFiles: qnaFiles.map((file) => ({ id: file.id, isEmpty: !file.qnaSections.length })),
});
luFileStatusStorage.publishAll(projectId);
qnaFileStatusStorage.publishAll(projectId);
diff --git a/Composer/packages/client/src/utils/buildUtil.ts b/Composer/packages/client/src/utils/buildUtil.ts
index 29958f92e0..8769f7ff50 100644
--- a/Composer/packages/client/src/utils/buildUtil.ts
+++ b/Composer/packages/client/src/utils/buildUtil.ts
@@ -2,137 +2,45 @@
// Licensed under the MIT License.
import { DialogInfo, LuFile } from '@bfc/shared';
-import keys from 'lodash/keys';
import { LuisConfig, QnaConfig } from '../constants';
import { getReferredLuFiles } from './luUtil';
import { getReferredQnaFiles } from './qnaUtil';
-import { getBaseName, getExtension } from './fileUtil';
+import { getBaseName } from './fileUtil';
-function createConfigId(fileId) {
- return `${fileId}.lu`;
+function createConfigId(fileId: string, language: string) {
+ return `${fileId}.${language}.lu`;
}
-function getLuFilesByDialogId(dialogId: string, luFiles: LuFile[]) {
- return luFiles.filter((lu) => getBaseName(lu.id) === dialogId).map((lu) => createConfigId(lu.id));
-}
-
-function getFileLocale(fileName: string) {
- //file name = 'a.en-us.lu'
- return getExtension(getBaseName(fileName));
-}
-//replace the dialogId with luFile's name
-function addLocaleToConfig(config: ICrossTrainConfig, luFiles: LuFile[]) {
- const { rootIds, triggerRules } = config;
- config.rootIds = rootIds.reduce((result: string[], id: string) => {
- return [...result, ...getLuFilesByDialogId(id, luFiles)];
- }, []);
- config.triggerRules = keys(triggerRules).reduce((result, key) => {
- const fileNames = getLuFilesByDialogId(key, luFiles);
- return {
- ...result,
- ...fileNames.reduce((result, name) => {
- const locale = getFileLocale(name);
- const triggers = triggerRules[key];
- keys(triggers).forEach((trigger) => {
- if (!result[name]) result[name] = {};
- const ids = triggers[trigger];
- if (Array.isArray(ids)) {
- result[name][trigger] = ids.map((id) => (id ? `${id}.${locale}.lu` : id));
- } else {
- result[name][trigger] = ids ? `${ids}.${locale}.lu` : ids;
- }
- });
- return result;
- }, {}),
- };
- }, {});
- return config;
-}
-interface ICrossTrainConfig {
- rootIds: string[];
- triggerRules: { [key: string]: any };
- intentName: string;
- verbose: boolean;
-}
+export function createCrossTrainConfig(dialogs: DialogInfo[], luFiles: LuFile[], languages: string[]) {
+ const config = dialogs.reduce((result, { isRoot: rootDialog, intentTriggers, id, luFile: luFileId }) => {
+ const luFile = luFiles.find((luFile) => getBaseName(luFile.id) === luFileId);
-//generate the cross-train config without locale
-/* the config is like
- {
- rootIds: [
- 'main.en-us.lu',
- 'main.fr-fr.lu'
- ],
- triggerRules: {
- 'main.en-us.lu': {
- 'dia1_trigger': 'dia1.en-us.lu',
- 'dia2_trigger': 'dia2.en-us.lu'
- },
- 'dia2.en-us.lu': {
- 'dia3_trigger': 'dia3.en-us.lu',
- 'dia4_trigger': 'dia4.en-us.lu'
- },
- 'main.fr-fr.lu': {
- 'dia1_trigger': 'dia1.fr-fr.lu'
- }
- },
- intentName: '_Interruption',
- verbose: true
- }
- */
-
-export function createCrossTrainConfig(dialogs: DialogInfo[], luFiles: LuFile[]): ICrossTrainConfig {
- const triggerRules = {};
- const countMap = {};
+ if (!luFile) return result;
- //map all referred lu files
- luFiles.forEach((file) => {
- countMap[getBaseName(file.id)] = 1;
- });
+ const filtered = intentTriggers.filter((intentTrigger) =>
+ luFile.intents.find((intent) => intent.Name === intentTrigger.intent || intentTrigger.intent === '')
+ );
- let rootId = '';
- dialogs.forEach((dialog) => {
- if (dialog.isRoot) rootId = dialog.id;
- const luFile = luFiles.find((luFile) => getBaseName(luFile.id) === dialog.luFile);
- if (luFile) {
- const fileId = dialog.id;
- const { intentTriggers } = dialog;
- // filter intenttrigger which be involved in lu file
- //find the trigger's dialog that use a recognizer
- intentTriggers
- .filter((intentTrigger) => luFile.intents.find((intent) => intent.Name === intentTrigger.intent))
- .forEach((item) => {
- //find all dialogs in trigger that has a luis recognizer
- const used = item.dialogs.filter((dialog) => !!countMap[dialog]);
+ if (!filtered.length) return result;
- const deduped = Array.from(new Set(used));
-
- const result = {};
- if (deduped.length === 1) {
- result[item.intent] = deduped[0];
- } else if (deduped.length) {
- result[item.intent] = deduped;
- } else {
- result[item.intent] = '';
- }
+ languages.forEach((language) => {
+ const triggers = filtered.reduce((result, { intent, dialogs }) => {
+ const ids = dialogs
+ .map((dialog) => createConfigId(dialog, language))
+ .filter((id) => luFiles.some((file) => `${file.id}.lu` === id));
+ if (!ids.length && dialogs.length) return result;
+ result[intent] = ids;
+ return result;
+ }, {});
+ result[createConfigId(id, language)] = { rootDialog, triggers };
+ });
- triggerRules[fileId] = { ...triggerRules[fileId], ...result };
- });
- }
- });
+ return result;
+ }, {});
- const crossTrainConfig: ICrossTrainConfig = {
- rootIds: [],
- triggerRules: {},
- intentName: '_Interruption',
- verbose: true,
- };
- crossTrainConfig.rootIds = keys(countMap).filter(
- (key) => (countMap[key] === 0 || key === rootId) && triggerRules[key]
- );
- crossTrainConfig.triggerRules = triggerRules;
- return addLocaleToConfig(crossTrainConfig, luFiles);
+ return config;
}
export function isBuildConfigComplete(config, dialogs, luFiles, qnaFiles) {
@@ -154,3 +62,5 @@ export function isBuildConfigComplete(config, dialogs, luFiles, qnaFiles) {
export function needsBuild(dialogs) {
return dialogs.some((dialog) => typeof dialog.content.recognizer === 'string');
}
+
+export function createRecognizerTypeMap(dialogs: DialogInfo[]) {}
diff --git a/Composer/packages/lib/indexers/package.json b/Composer/packages/lib/indexers/package.json
index 1cabcd5d49..89f5186a10 100644
--- a/Composer/packages/lib/indexers/package.json
+++ b/Composer/packages/lib/indexers/package.json
@@ -26,7 +26,7 @@
"rimraf": "^2.6.3"
},
"dependencies": {
- "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8",
+ "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",
"adaptive-expressions": "4.10.0-preview-147186",
"botbuilder-lg": "^4.10.0-preview-150886",
"lodash": "^4.17.19"
diff --git a/Composer/packages/lib/indexers/src/dialogIndexer.ts b/Composer/packages/lib/indexers/src/dialogIndexer.ts
index de2a2ea730..2e88ecfb5e 100644
--- a/Composer/packages/lib/indexers/src/dialogIndexer.ts
+++ b/Composer/packages/lib/indexers/src/dialogIndexer.ts
@@ -19,6 +19,7 @@ import { JsonWalk, VisitorFunc } from './utils/jsonWalk';
import { getBaseName } from './utils/help';
import extractIntentTriggers from './dialogUtils/extractIntentTriggers';
import { createPath } from './validations/expressionValidation/utils';
+
// find out all lg templates given dialog
function extractLgTemplates(id, dialog): LgTemplateJsonPath[] {
const templates: LgTemplateJsonPath[] = [];
diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json
index 7027248de7..2409338c40 100644
--- a/Composer/packages/server/package.json
+++ b/Composer/packages/server/package.json
@@ -62,9 +62,9 @@
"@bfc/lg-languageserver": "*",
"@bfc/lu-languageserver": "*",
"@bfc/shared": "*",
- "@microsoft/bf-dispatcher": "^4.10.0-preview.141651",
+ "@microsoft/bf-dispatcher": "^4.11.0-beta.20201015.008a3a4",
"@microsoft/bf-generate-library": "^4.10.0-daily.20201008.172736",
- "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8",
+ "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",
"archiver": "^5.0.2",
"axios": "^0.19.2",
"azure-storage": "^2.10.3",
diff --git a/Composer/packages/server/src/controllers/__tests__/project.test.ts b/Composer/packages/server/src/controllers/__tests__/project.test.ts
index d7f52366d2..d4be267466 100644
--- a/Composer/packages/server/src/controllers/__tests__/project.test.ts
+++ b/Composer/packages/server/src/controllers/__tests__/project.test.ts
@@ -367,17 +367,12 @@ describe('publish luis files', () => {
body: {
authoringKey: '0d4991873f334685a9686d1b48e0ff48',
projectId: projectId,
- crossTrainConfig: {
- rootIds: ['bot1.en-us.lu'],
- triggerRules: { 'bot1.en-us.lu': {} },
- intentName: '_Interruption',
- verbose: true,
- },
+ crossTrainConfig: {},
luFiles: [],
},
} as Request;
await ProjectController.build(mockReq, mockRes);
- expect(mockRes.status).toHaveBeenCalledWith(200);
+ expect(mockRes.status).toHaveBeenCalled();
});
});
diff --git a/Composer/packages/server/src/controllers/project.ts b/Composer/packages/server/src/controllers/project.ts
index 562574c7c3..e550fa38cc 100644
--- a/Composer/packages/server/src/controllers/project.ts
+++ b/Composer/packages/server/src/controllers/project.ts
@@ -326,13 +326,14 @@ async function build(req: Request, res: Response) {
const currentProject = await BotProjectService.getProjectById(projectId, user);
if (currentProject !== undefined) {
try {
- const { luisConfig, qnaConfig, luFiles, qnaFiles, crossTrainConfig } = req.body;
+ const { luisConfig, qnaConfig, luFiles, qnaFiles, crossTrainConfig, recognizerTypes } = req.body;
const files = await currentProject.buildFiles({
luisConfig,
qnaConfig,
- luFileIds: luFiles,
- qnaFileIds: qnaFiles,
+ luResource: luFiles,
+ qnaResource: qnaFiles,
crossTrainConfig,
+ recognizerTypes,
});
res.status(200).json(files);
} catch (error) {
diff --git a/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts b/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts
index ead0964b84..d703273b73 100644
--- a/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts
+++ b/Composer/packages/server/src/models/bot/__tests__/botProject.test.ts
@@ -11,6 +11,9 @@ import { Path } from '../../../utility/path';
import { BotProject } from '../botProject';
import { LocationRef } from '../interface';
+import { Resource } from './../interface';
+import { RecognizerTypes } from './../recognizer';
+
jest.mock('azure-storage', () => {
return {};
});
@@ -287,16 +290,19 @@ describe('buildFiles', () => {
qnaRegion: 'westus',
subscriptionKey: '21640b8e2110449abfdfccf2f6bbee02',
};
- const luFileIds = ['a.en-us', 'b.en-us', 'bot1.en-us'];
- const qnaFileIds = ['a.en-us', 'b.en-us', 'bot1.en-us'];
- const crossTrainConfig = {
- botName: 'bot1',
- rootIds: [],
- triggerRules: {},
- intentName: '_Interruption',
- verbose: true,
- };
- await proj.buildFiles({ luisConfig, qnaConfig, luFileIds, qnaFileIds, crossTrainConfig });
+ const luResource: Resource[] = [
+ { id: 'a.en-us', isEmpty: false },
+ { id: 'b.en-us', isEmpty: false },
+ { id: 'bot1.en-us', isEmpty: false },
+ ];
+ const qnaResource: Resource[] = [
+ { id: 'a.en-us', isEmpty: false },
+ { id: 'b.en-us', isEmpty: false },
+ { id: 'bot1.en-us', isEmpty: false },
+ ];
+ const crossTrainConfig = {};
+ const recognizerTypes: RecognizerTypes = { a: 'DefaultRecognizer', b: 'DefaultRecognizer', c: 'DefaultRecognizer' };
+ await proj.buildFiles({ luisConfig, qnaConfig, luResource, qnaResource, crossTrainConfig, recognizerTypes });
try {
if (fs.existsSync(path)) {
diff --git a/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts b/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts
new file mode 100644
index 0000000000..c39013199d
--- /dev/null
+++ b/Composer/packages/server/src/models/bot/__tests__/preBuilder.test.ts
@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { FileInfo } from '@bfc/shared';
+
+import { PreBuilder } from './../preBuilder';
+import { CrossTrainConfig } from './../builder';
+
+const storage: any = {};
+const builder = new PreBuilder('c:/bot', storage);
+
+describe('Test the crossTrain config change', () => {
+ it('should convert the cross train config from id to relativePath', () => {
+ const configObject: CrossTrainConfig = {
+ 'main.en-us.lu': {
+ rootDialog: true,
+ triggers: {
+ dia1Trigger: ['dia1.en-us.lu'],
+ dia2Trigger: ['dia2.en-us.lu'],
+ },
+ },
+ 'dia2.en-us.lu': {
+ rootDialog: false,
+ triggers: {
+ dia3Trigger: ['dia3.en-us.lu'],
+ dia4Trigger: ['dia4.en-us.lu'],
+ },
+ },
+ 'main.fr-fr.lu': {
+ rootDialog: true,
+ triggers: {
+ dia1Trigger: ['dia1.fr-fr.lu'],
+ },
+ },
+ };
+
+ const files: FileInfo[] = [
+ {
+ name: 'main.en-us.lu',
+ content: '',
+ path: 'c:/bot/lu/en-us/main.en-us.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ {
+ name: 'dia1.en-us.lu',
+ content: '',
+ path: 'c:/bot/dia1/lu/en-us/dia1.en-us.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ {
+ name: 'dia2.en-us.lu',
+ content: '',
+ path: 'c:/bot/dia2/lu/en-us/dia2.en-us.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ {
+ name: 'dia3.en-us.lu',
+ content: '',
+ path: 'c:/bot/dia3/lu/en-us/dia3.en-us.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ {
+ name: 'dia4.en-us.lu',
+ content: '',
+ path: 'c:/bot/dia4/lu/en-us/dia4.en-us.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ { name: 'main.fr-fr.lu', content: '', path: 'c:/bot/lu/fr-fr/main.fr-fr.lu', relativePath: '', lastModified: '' },
+ {
+ name: 'dia1.fr-fr.lu',
+ content: '',
+ path: 'c:/bot/dia1/lu/fr-fr/dia1.fr-fr.lu',
+ relativePath: '',
+ lastModified: '',
+ },
+ ];
+ const result = builder.generateCrossTrainConfig(configObject, files);
+ expect(result['../dia2/lu/en-us/dia2.en-us.lu']).not.toBeUndefined();
+ expect(result['../dia2/lu/en-us/dia2.en-us.lu'].triggers.dia3Trigger[0]).toBe('../dia3/lu/en-us/dia3.en-us.lu');
+ });
+});
diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts
index 72f21cefd9..df8d59847c 100644
--- a/Composer/packages/server/src/models/bot/botProject.ts
+++ b/Composer/packages/server/src/models/bot/botProject.ts
@@ -35,6 +35,7 @@ import { IFileStorage } from './../storage/interface';
import { LocationRef, IBuildConfig } from './interface';
import { retrieveSkillManifests } from './skillManager';
import { defaultFilePath, serializeFiles, parseFileName } from './botStructure';
+import { PreBuilder } from './preBuilder';
const debug = log.extend('bot-project');
const mkDirAsync = promisify(fs.mkdir);
@@ -55,6 +56,7 @@ export class BotProject implements IBotProject {
public dataDir: string;
public fileStorage: IFileStorage;
public builder: Builder;
+ public preBuilder: PreBuilder;
public defaultSDKSchema: {
[key: string]: string;
};
@@ -80,6 +82,7 @@ export class BotProject implements IBotProject {
this.settingManager = new DefaultSettingManager(this.dir);
this.fileStorage = StorageService.getStorageClient(this.ref.storageId, user);
this.builder = new Builder(this.dir, this.fileStorage, defaultLanguage);
+ this.preBuilder = new PreBuilder(this.dir, this.fileStorage);
}
public get dialogFiles() {
@@ -459,25 +462,34 @@ export class BotProject implements IBotProject {
public buildFiles = async ({
luisConfig,
qnaConfig,
- luFileIds = [],
- qnaFileIds = [],
+ luResource = [],
+ qnaResource = [],
crossTrainConfig,
+ recognizerTypes,
}: IBuildConfig) => {
- if ((luFileIds.length || qnaFileIds.length) && this.settings) {
+ if (this.settings) {
+ const emptyFiles = {};
const luFiles: FileInfo[] = [];
- luFileIds.forEach((id) => {
- const f = this.files.get(`${id}.lu`);
+ luResource.forEach(({ id, isEmpty }) => {
+ const fileName = `${id}.lu`;
+ const f = this.files.get(fileName);
if (f) {
luFiles.push(f);
+ emptyFiles[fileName] = isEmpty;
}
});
const qnaFiles: FileInfo[] = [];
- qnaFileIds.forEach((id) => {
- const f = this.files.get(`${id}.qna`);
+ qnaResource.forEach(({ id, isEmpty }) => {
+ const fileName = `${id}.qna`;
+ const f = this.files.get(fileName);
if (f) {
qnaFiles.push(f);
+ emptyFiles[fileName] = isEmpty;
}
});
+
+ await this.preBuilder.prebuild(recognizerTypes, { crossTrainConfig, luFiles, qnaFiles, emptyFiles });
+
this.builder.setBuildConfig(
{ ...luisConfig, subscriptionKey: qnaConfig.subscriptionKey, qnaRegion: qnaConfig.qnaRegion },
crossTrainConfig,
@@ -730,7 +742,10 @@ export class BotProject implements IBotProject {
// load only from the data dir, otherwise may get "build" versions from
// deployment process
const root = this.dataDir;
- const paths = await this.fileStorage.glob([pattern, '!(generated/**)', '!(runtime/**)'], root);
+ const paths = await this.fileStorage.glob(
+ [pattern, '!(generated/**)', '!(runtime/**)', '!(recognizers/**)'],
+ root
+ );
for (const filePath of paths.sort()) {
const realFilePath: string = Path.join(root, filePath);
diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts
index 142b546b42..65b5987a93 100644
--- a/Composer/packages/server/src/models/bot/builder.ts
+++ b/Composer/packages/server/src/models/bot/builder.ts
@@ -1,60 +1,50 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
+/* eslint-disable @typescript-eslint/no-var-requires */
import { FileInfo, IConfig } from '@bfc/shared';
+import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler';
+import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler';
import { Path } from '../../utility/path';
import { IFileStorage } from '../storage/interface';
import log from '../../logger';
-import { luImportResolverGenerator, getLUFiles, getQnAFiles } from './luResolver';
-import { ComposerReservoirSampler } from './sampler/ReservoirSampler';
-import { ComposerBootstrapSampler } from './sampler/BootstrapSampler';
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
const crossTrainer = require('@microsoft/bf-lu/lib/parser/cross-train/crossTrainer.js');
-// eslint-disable-next-line @typescript-eslint/no-var-requires
const luBuild = require('@microsoft/bf-lu/lib/parser/lubuild/builder.js');
-// eslint-disable-next-line @typescript-eslint/no-var-requires
const qnaBuild = require('@microsoft/bf-lu/lib/parser/qnabuild/builder.js');
-// eslint-disable-next-line @typescript-eslint/no-var-requires
const LuisBuilder = require('@microsoft/bf-lu/lib/parser/luis/luisBuilder');
-// eslint-disable-next-line @typescript-eslint/no-var-requires
const luisToLuContent = require('@microsoft/bf-lu/lib/parser/luis/luConverter');
const GENERATEDFOLDER = 'generated';
const INTERUPTION = 'interuption';
+const SAMPLE_SIZE_CONFIGURATION = 2;
-export interface ICrossTrainConfig {
- rootIds: string[];
- triggerRules: { [key: string]: any };
- intentName: string;
- verbose: boolean;
- botName: string;
-}
+export type SingleConfig = {
+ rootDialog: boolean;
+ triggers: {
+ [key: string]: string[];
+ };
+};
+
+export type CrossTrainConfig = {
+ [key: string]: SingleConfig;
+};
-export interface IDownSamplingConfig {
+export type DownSamplingConfig = {
maxImbalanceRatio: number;
maxUtteranceAllowed: number;
-}
+};
export class Builder {
public botDir: string;
- public dialogsDir: string;
public generatedFolderPath: string;
public interruptionFolderPath: string;
public storage: IFileStorage;
public config: IConfig | null = null;
- public downSamplingConfig: IDownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 };
+ public downSamplingConfig: DownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 };
private _locale: string;
-
- public crossTrainConfig: ICrossTrainConfig = {
- rootIds: [],
- triggerRules: {},
- intentName: '_Interruption',
- verbose: true,
- botName: '',
- };
+ public crossTrainConfig: CrossTrainConfig = {};
private luBuilder = new luBuild.Builder((message) => {
log(message);
@@ -65,8 +55,7 @@ export class Builder {
constructor(path: string, storage: IFileStorage, locale: string) {
this.botDir = path;
- this.dialogsDir = this.botDir;
- this.generatedFolderPath = Path.join(this.dialogsDir, GENERATEDFOLDER);
+ this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER);
this.interruptionFolderPath = Path.join(this.generatedFolderPath, INTERUPTION);
this.storage = storage;
this._locale = locale;
@@ -75,15 +64,11 @@ export class Builder {
public build = async (luFiles: FileInfo[], qnaFiles: FileInfo[], allFiles: FileInfo[]) => {
try {
await this.createGeneratedDir();
-
//do cross train before publish
await this.crossTrain(luFiles, qnaFiles, allFiles);
const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles();
await this.runLuBuild(interruptionLuFiles, allFiles);
await this.runQnaBuild(interruptionQnaFiles);
-
- //remove the cross train result
- await this.cleanCrossTrain();
} catch (error) {
throw new Error(error.message ?? error.text ?? 'Error publishing to LUIS or QNA.');
}
@@ -99,9 +84,9 @@ export class Builder {
}
};
- public setBuildConfig(config: IConfig, crossTrainConfig: ICrossTrainConfig, downSamplingConfig: IDownSamplingConfig) {
+ public setBuildConfig(config: IConfig, crossTrainConfig: CrossTrainConfig, downSamplingConfig: DownSamplingConfig) {
this.config = config;
- this.crossTrainConfig = { ...crossTrainConfig, botName: this.config.name };
+ this.crossTrainConfig = crossTrainConfig;
this.downSamplingConfig = downSamplingConfig;
}
@@ -116,6 +101,8 @@ export class Builder {
private async createGeneratedDir() {
// clear previous folder
await this.deleteDir(this.generatedFolderPath);
+ //remove the cross train result
+ await this.cleanCrossTrain();
await this.storage.mkDir(this.generatedFolderPath);
}
@@ -127,8 +114,8 @@ export class Builder {
const qnaContents = qnaFiles.map((file) => {
return { content: file.content, id: file.name };
});
- const resolver = luImportResolverGenerator([...getLUFiles(allFiles), ...getQnAFiles(allFiles)]);
- const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, resolver);
+
+ const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, {});
await this.writeFiles(result.luResult);
await this.writeFiles(result.qnaResult);
@@ -163,7 +150,8 @@ export class Builder {
//do bootstramp sampling to make the utterances' number ratio to 1:10
const bootstrapSampler = new ComposerBootstrapSampler(
luObject.utterances,
- this.downSamplingConfig.maxImbalanceRatio
+ this.downSamplingConfig.maxImbalanceRatio,
+ SAMPLE_SIZE_CONFIGURATION
);
luObject.utterances = bootstrapSampler.getSampledUtterances();
//if detect the utterances>15000, use reservoir sampling to down size
@@ -178,10 +166,14 @@ export class Builder {
private async downsizeUtterances(luContents: any) {
return await Promise.all(
luContents.map(async (luContent) => {
- const result = await LuisBuilder.fromLUAsync(luContent.content);
- const sampledResult = this.doDownSampling(result);
- const content = luisToLuContent(sampledResult);
- return { ...luContent, content };
+ if (luContent.content) {
+ const result = await LuisBuilder.fromLUAsync(luContent.content);
+ const sampledResult = this.doDownSampling(result);
+ const content = luisToLuContent(sampledResult);
+ return { ...luContent, content };
+ }
+
+ return luContent;
})
);
}
@@ -201,69 +193,47 @@ export class Builder {
private async runLuBuild(files: FileInfo[], allFiles: FileInfo[]) {
const config = await this._getConfig(files, 'lu');
- const resolver = luImportResolverGenerator(getLUFiles(allFiles));
- const loadResult = await this.luBuilder.loadContents(
- config.models,
- config.fallbackLocal,
- config.suffix,
- config.region,
- null,
- resolver
- );
- loadResult.luContents = await this.downsizeUtterances(loadResult.luContents);
+
+ let luContents = await this.luBuilder.loadContents(config.models, {
+ culture: config.fallbackLocal,
+ });
+
+ luContents = await this.downsizeUtterances(luContents);
const authoringEndpoint = config.authoringEndpoint ?? `https://${config.region}.api.cognitive.microsoft.com`;
- const buildResult = await this.luBuilder.build(
- loadResult.luContents,
- loadResult.recognizers,
- config.authoringKey,
- authoringEndpoint,
- config.botName,
- config.suffix,
- config.fallbackLocal,
- true,
- false,
- loadResult.multiRecognizers,
- loadResult.settings,
- loadResult.crosstrainedRecognizers,
- 'crosstrained'
- );
- await this.luBuilder.writeDialogAssets(buildResult, true, this.generatedFolderPath);
+ const buildResult = await this.luBuilder.build(luContents, config.authoringKey, config.botName, {
+ endpoint: authoringEndpoint,
+ suffix: config.suffix,
+ keptVersionCount: 10,
+ isStaging: false,
+ });
+
+ await this.luBuilder.writeDialogAssets(buildResult, {
+ force: true,
+ out: this.generatedFolderPath,
+ });
}
private async runQnaBuild(files: FileInfo[]) {
const config = await this._getConfig(files, 'qna');
- // if (config.models.length === 0) {
- // throw new Error('No QnA files exist');
- // }
-
- const loadResult = await this.qnaBuilder.loadContents(
- config.models,
- config.botName,
- config.suffix,
- config.qnaRegion,
- config.fallbackLocal
- );
- if (loadResult.qnaContents) {
+
+ const qnaContents = await this.qnaBuilder.loadContents(config.models, {
+ culture: config.fallbackLocal,
+ });
+
+ if (qnaContents) {
const subscriptionKeyEndpoint =
config.endpoint ?? `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`;
- const buildResult = await this.qnaBuilder.build(
- loadResult.qnaContents,
- loadResult.recognizers,
- config.subscriptionKey,
- subscriptionKeyEndpoint,
- config.botName,
- config.suffix,
- config.fallbackLocal,
- loadResult.multiRecognizers,
- loadResult.settings,
- loadResult.crosstrainedRecognizers,
- 'crosstrained'
- );
- await this.qnaBuilder.writeDialogAssets(buildResult, true, this.generatedFolderPath);
- } else {
- await this.qnaBuilder.writeDialogAssets(loadResult.crosstrainedRecognizer.save(), true, this.generatedFolderPath);
+ const buildResult = await this.qnaBuilder.build(qnaContents, config.subscriptionKey, config.botName, {
+ endpoint: subscriptionKeyEndpoint,
+ suffix: config.suffix,
+ });
+
+ await this.qnaBuilder.writeDialogAssets(buildResult, {
+ force: true,
+ out: this.generatedFolderPath,
+ });
}
}
diff --git a/Composer/packages/server/src/models/bot/interface.ts b/Composer/packages/server/src/models/bot/interface.ts
index e8d28fb959..4313fb01da 100644
--- a/Composer/packages/server/src/models/bot/interface.ts
+++ b/Composer/packages/server/src/models/bot/interface.ts
@@ -3,7 +3,10 @@
import { ILuisConfig, IQnAConfig } from '@bfc/shared';
-import { ICrossTrainConfig } from './builder';
+import { CrossTrainConfig } from './builder';
+import { RecognizerTypes } from './recognizer';
+
+export type Resource = { id: string; isEmpty: boolean };
export interface LocationRef {
storageId: string;
@@ -13,9 +16,10 @@ export interface LocationRef {
export interface IBuildConfig {
luisConfig: ILuisConfig;
qnaConfig: IQnAConfig;
- luFileIds: string[];
- qnaFileIds: string[];
- crossTrainConfig: ICrossTrainConfig;
+ luResource: Resource[];
+ qnaResource: Resource[];
+ crossTrainConfig: CrossTrainConfig;
+ recognizerTypes: RecognizerTypes;
}
export interface ILuisSettings {
diff --git a/Composer/packages/server/src/models/bot/preBuilder.ts b/Composer/packages/server/src/models/bot/preBuilder.ts
new file mode 100644
index 0000000000..a13492122e
--- /dev/null
+++ b/Composer/packages/server/src/models/bot/preBuilder.ts
@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { FileInfo } from '@bfc/shared';
+import keys from 'lodash/keys';
+
+import { IFileStorage } from '../storage/interface';
+
+import { Path } from './../../utility/path';
+import { CrossTrainConfig } from './builder';
+import recognizers, { RecognizerTypes } from './recognizer';
+
+const RECOGNIZERS = 'recognizers';
+
+export class PreBuilder {
+ folderPath: string;
+ storage: IFileStorage;
+
+ constructor(botDir: string, storage: IFileStorage) {
+ this.folderPath = Path.join(botDir, RECOGNIZERS);
+ this.storage = storage;
+ }
+
+ public async prebuild(
+ recognizerTypes: RecognizerTypes,
+ options: {
+ crossTrainConfig?: CrossTrainConfig;
+ luFiles: FileInfo[];
+ qnaFiles: FileInfo[];
+ emptyFiles: { [fileName: string]: boolean };
+ }
+ ) {
+ await this.createRecognizersDir();
+
+ await this.updateCrossTrainConfig(options.luFiles, options.crossTrainConfig);
+
+ await this.updateRecognizers(recognizerTypes, [...options.luFiles, ...options.qnaFiles], options.emptyFiles);
+ }
+
+ async createRecognizersDir() {
+ if (!(await this.storage.exists(this.folderPath))) {
+ await this.storage.mkDir(this.folderPath);
+ }
+ }
+
+ async updateCrossTrainConfig(luFiles: FileInfo[], crossTrainConfig?: CrossTrainConfig) {
+ if (crossTrainConfig && luFiles.length) {
+ const configWithPath = this.generateCrossTrainConfig(crossTrainConfig, luFiles);
+ await this.storage.writeFile(
+ `${this.folderPath}/cross-train.config.json`,
+ JSON.stringify(configWithPath, null, 2)
+ );
+ }
+ }
+
+ replaceCrossTrainId(id: string, files: FileInfo[]) {
+ if (!id) return id;
+ const luFile = files.find((item) => item.name === id);
+ return Path.relative(this.folderPath, luFile?.path ?? '');
+ }
+
+ /**
+ * convert the cross train config from id to relativePath. The cli use the config to find the files.
+ * config = {
+ * 'main.lu': {
+ * rootDialog: true,
+ * triggers: {
+ * 'intentA':'diaA.lu',
+ * 'intentB': 'diaB.lu'
+ * }
+ * }
+ * }
+ */
+ generateCrossTrainConfig(crossTrainConfig: CrossTrainConfig, files: FileInfo[]) {
+ const pathCache = {};
+
+ const configWithPath = keys(crossTrainConfig).reduce((result: CrossTrainConfig, key: string) => {
+ const { triggers: preTriggers, rootDialog } = crossTrainConfig[key];
+ // replace the key with path
+ if (!pathCache[key]) pathCache[key] = this.replaceCrossTrainId(key, files);
+
+ const triggers = keys(preTriggers).reduce((result: { [key: string]: string[] }, key) => {
+ const ids = preTriggers[key];
+ result[key] = ids.map((item) => {
+ // replace the trigger value with path
+ if (!pathCache[item]) pathCache[item] = this.replaceCrossTrainId(item, files);
+
+ return pathCache[item];
+ });
+ return result;
+ }, {});
+
+ result[pathCache[key]] = { triggers, rootDialog };
+ return result;
+ }, {});
+
+ return configWithPath;
+ }
+
+ /**
+ * update the recoginzers before build
+ */
+ async updateRecognizers(recognizerTypes: RecognizerTypes, files: FileInfo[], emptyFiles) {
+ await Promise.all(
+ keys(recognizerTypes).map(async (item) => {
+ const type = recognizerTypes[item];
+ const targetFiles = files
+ .filter((file) => file.name.startsWith(item) && !emptyFiles[file.name])
+ .map((item) => item.name);
+
+ const updateFunc = recognizers[type] ?? recognizers.Default;
+ await updateFunc(item, targetFiles, this.storage, {
+ defalutLanguage: 'en-us',
+ folderPath: this.folderPath,
+ });
+ })
+ );
+ }
+}
diff --git a/Composer/packages/server/src/models/bot/recognizer.ts b/Composer/packages/server/src/models/bot/recognizer.ts
new file mode 100644
index 0000000000..8bf8181f02
--- /dev/null
+++ b/Composer/packages/server/src/models/bot/recognizer.ts
@@ -0,0 +1,175 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { SDKKinds } from '@bfc/shared';
+
+import { IFileStorage } from '../storage/interface';
+
+export type UpdateRecognizer = (
+ target: string,
+ fileNames: string[],
+ storage: IFileStorage,
+ options: { defalutLanguage?: string; folderPath?: string }
+) => Promise | void;
+
+export type RecognizerType = SDKKinds.CrossTrainedRecognizerSet | SDKKinds.LuisRecognizer | string;
+export type RecognizerTypes = { [fileName: string]: RecognizerType };
+
+type GeneratedDialog = { name: string; content: string };
+
+const LuisRecognizerTemplate = (target: string, fileName: string) => ({
+ $kind: SDKKinds.LuisRecognizer,
+ id: `LUIS_${target}`,
+ applicationId: `=settings.luis.${fileName.replace(/[.-]/g, '_')}.appId`,
+ version: `=settings.luis.${fileName.replace(/[.-]/g, '_')}.version`,
+ endpoint: '=settings.luis.endpoint',
+ endpointKey: '=settings.luis.endpointKey',
+});
+
+const MultiLanguageRecognizerTemplate = (target: string) => ({
+ $kind: SDKKinds.MultiLanguageRecognizer,
+ id: `LUIS_${target}`,
+ recognizers: {},
+});
+
+const CrossTrainedRecognizerTemplate = (): {
+ $kind: string;
+ recognizers: string[];
+} => ({
+ $kind: SDKKinds.CrossTrainedRecognizerSet,
+ recognizers: [],
+});
+
+//in composer the luFile name is a.local.lu
+const getLuFileLocal = (fileName: string) => {
+ const items = fileName.split('.');
+ return items[items.length - 2];
+};
+
+const getMultiLanguagueRecognizerDialog = (target: string, luFileNames: string[], defalutLanguage = 'en-us') => {
+ const multiLanguageRecognizer = MultiLanguageRecognizerTemplate(target);
+
+ luFileNames.forEach((item) => {
+ const local = getLuFileLocal(item);
+ multiLanguageRecognizer.recognizers[local] = item;
+ if (local === defalutLanguage) {
+ multiLanguageRecognizer.recognizers[''] = item;
+ }
+ });
+
+ return { name: `${target}.lu.dialog`, content: JSON.stringify(multiLanguageRecognizer, null, 2) };
+};
+
+const getCrossTrainedRecognizerDialog = (target: string, fileNames: string[]) => {
+ const crossTrainedRecognizer = CrossTrainedRecognizerTemplate();
+
+ if (fileNames.some((item) => item.endsWith('.qna'))) {
+ crossTrainedRecognizer.recognizers.push(`${target}.qna`);
+ }
+
+ if (fileNames.some((item) => item.endsWith('.lu'))) {
+ crossTrainedRecognizer.recognizers.push(`${target}.lu`);
+ }
+
+ return {
+ name: `${target}.lu.qna.dialog`,
+ content: JSON.stringify(crossTrainedRecognizer, null, 2),
+ };
+};
+
+const getLuisRecognizerDialogs = (target: string, luFileNames: string[]) => {
+ return luFileNames.map((item) => {
+ const local = getLuFileLocal(item);
+ return {
+ name: `${target}.${local}.lu.dialog`,
+ content: JSON.stringify(LuisRecognizerTemplate(target, item), null, 2),
+ };
+ });
+};
+
+/**
+ * DefaultRecognizer:
+ * luisRecoginzers: create and preserve(exists)
+ * multiLanguageRecognizer: update the recognizers
+ * crossTrainedRecognizer: update the recognizers
+ *
+ * @param target the dialog Id
+ * @param fileNames the lu and qna files name list
+ * @param folderPath the recognizers folder's path
+ */
+const updateRecognizers = (isCrosstrain: boolean): UpdateRecognizer => async (
+ target: string,
+ fileNames: string[],
+ storage: IFileStorage,
+ { defalutLanguage, folderPath }
+) => {
+ const luFileNames = fileNames.filter((item) => item.endsWith('.lu'));
+ const multiLanguageRecognizerDialog = getMultiLanguagueRecognizerDialog(target, luFileNames, defalutLanguage);
+ const luisRecognizersDialogs = getLuisRecognizerDialogs(target, luFileNames);
+ const needUpdateDialogs: GeneratedDialog[] = [];
+ const needPreserveDialogs: GeneratedDialog[] = [];
+
+ if (isCrosstrain) {
+ const crossTrainedRecognizerDialog = getCrossTrainedRecognizerDialog(target, fileNames);
+ needUpdateDialogs.push(crossTrainedRecognizerDialog);
+ }
+
+ luisRecognizersDialogs.forEach((item) => {
+ needPreserveDialogs.push(item);
+ });
+
+ if (luisRecognizersDialogs.length) {
+ needUpdateDialogs.push(multiLanguageRecognizerDialog);
+ }
+
+ const previousFilePaths = await storage.glob(`${target}.*`, folderPath ?? '');
+ const currentFiles = [...needUpdateDialogs, ...needPreserveDialogs];
+
+ //if remove a local, need to delete these files
+ const needDeleteFiles = previousFilePaths.filter((item) => !currentFiles.some((file) => file.name === item));
+ const needupdateFiles = needUpdateDialogs.concat(
+ needPreserveDialogs.filter((item) => !previousFilePaths.some((path) => path === item.name))
+ );
+
+ await Promise.all(
+ needDeleteFiles.map(async (fileName) => {
+ return await storage.removeFile(`${folderPath}/${fileName}`);
+ })
+ );
+
+ await Promise.all(
+ needupdateFiles.map(async (item) => {
+ return await storage.writeFile(`${folderPath}/${item.name}`, item.content);
+ })
+ );
+};
+
+export const updateCrossTrained: UpdateRecognizer = updateRecognizers(true);
+
+export const updateLuis: UpdateRecognizer = updateRecognizers(false);
+
+/**
+ * RegexRecognizer now remove all the files
+ * ToDo: CustomRecognizer now remove all the files
+ */
+export const removeRecognizers: UpdateRecognizer = async (
+ target: string,
+ fileNames: string[],
+ storage: IFileStorage,
+ { folderPath }
+) => {
+ const filePaths = await storage.glob(`${target}.*`, folderPath ?? '');
+
+ await Promise.all(
+ filePaths.map(async (fileName) => {
+ return await storage.removeFile(`${folderPath}/${fileName}`);
+ })
+ );
+};
+
+const recognizers: { [key in RecognizerType]: UpdateRecognizer } = {
+ 'Microsoft.CrossTrainedRecognizerSet': updateCrossTrained,
+ 'Microsoft.LuisRecognizer': updateLuis,
+ Default: removeRecognizers,
+};
+
+export default recognizers;
diff --git a/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts b/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts
deleted file mode 100644
index 9170e64186..0000000000
--- a/Composer/packages/server/src/models/bot/sampler/BootstrapSampler.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { BootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/BootstrapSampler';
-import { Utility } from '@microsoft/bf-dispatcher/lib/utility/Utility';
-
-const SAMPLE_SIZE_CONFIGURATION = 2;
-
-export interface IUtterance {
- text: string;
- intent: string;
- entities: any[];
-}
-
-export class ComposerBootstrapSampler extends BootstrapSampler {
- private _maxImbalanceRatio: number;
- private _utterances: IUtterance[] = [];
-
- public constructor(utterances: IUtterance[], maxImbalanceRatio: number) {
- super({}, true, SAMPLE_SIZE_CONFIGURATION);
- this._utterances = utterances;
- this._maxImbalanceRatio = maxImbalanceRatio;
- utterances.forEach((e, index) => {
- const { intent } = e;
- this.addInstance(intent, index);
- });
- }
-
- public computeMaxBalanceNumber(): number {
- const numberInstancesPerLabelReduce: number = this.labels.reduce(
- (mini: number, key: string) => (this.instances[key].length < mini ? this.instances[key].length : mini),
- Number.MAX_SAFE_INTEGER
- );
-
- return this._maxImbalanceRatio * numberInstancesPerLabelReduce;
- }
-
- public computeSamplingNumberInstancesPerLabel(label = ''): number {
- return this.computeMaxBalanceNumber() * SAMPLE_SIZE_CONFIGURATION;
- }
-
- public getSampledUtterances() {
- if (this._maxImbalanceRatio) {
- this.resetLabelsAndMap();
-
- const sampledIndexes = this.sampleInstances();
-
- const set = new Set([...sampledIndexes]);
-
- return Array.from(set).map((index) => this._utterances[index]);
- } else {
- return this._utterances;
- }
- }
-
- //do re-sample if the ratio is beigher than the maxImbalanceRatio
- public *sampleInstances() {
- for (const key in this.instances) {
- const instanceArray: number[] = this.instances[key];
- const numberInstancesPerLabel: number = instanceArray.length;
- const maxBalanceNumber: number = this.computeMaxBalanceNumber();
- if (numberInstancesPerLabel > maxBalanceNumber) {
- const numberSamplingInstancesPerLabel: number = this.computeSamplingNumberInstancesPerLabel(key);
- for (let i = 0; i < numberSamplingInstancesPerLabel; i++) {
- const indexRandom = Utility.getRandomInt(numberInstancesPerLabel);
- yield instanceArray[indexRandom];
- }
- } else {
- for (let i = 0; i < numberInstancesPerLabel; i++) {
- yield instanceArray[i];
- }
- }
- }
- }
-}
diff --git a/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts b/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts
deleted file mode 100644
index d777ec4773..0000000000
--- a/Composer/packages/server/src/models/bot/sampler/ReservoirSampler.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-import { ReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ReservoirSampler';
-
-import { IUtterance } from './BootstrapSampler';
-
-export class ComposerReservoirSampler extends ReservoirSampler {
- private _utterances: IUtterance[] = [];
- private _maxUtteranceAllowed: number;
-
- public constructor(utterances: IUtterance[], maxUtteranceAllowed: number) {
- super({});
- this._utterances = utterances;
- this._maxUtteranceAllowed = maxUtteranceAllowed;
- utterances.forEach((e, index) => {
- this.addInstance(e.intent, index);
- });
- }
-
- public getSampledUtterances() {
- if (this._maxUtteranceAllowed && this._utterances.length > this._maxUtteranceAllowed) {
- this.resetLabelsAndMap();
-
- const sampledIndexes = this.sampleInstances(this._maxUtteranceAllowed);
-
- const set = new Set([...sampledIndexes]);
-
- return Array.from(set).map((index) => this._utterances[index]);
- } else {
- return this._utterances;
- }
- }
-}
diff --git a/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts b/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts
deleted file mode 100644
index 94d0577fbb..0000000000
--- a/Composer/packages/server/src/models/bot/sampler/__tests__/BootstrapSampler.test.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-import { ComposerBootstrapSampler } from '../BootstrapSampler';
-
-describe('BootstrapSampler', () => {
- it('balence the utterances ratio in intents after bootstrap sampling', async () => {
- const utterances = [
- { intent: '0', text: '1', entities: [] },
- { intent: '0', text: '2', entities: [] },
- { intent: '1', text: '3', entities: [] },
- { intent: '1', text: '4', entities: [] },
- { intent: '1', text: '5', entities: [] },
- { intent: '1', text: '6', entities: [] },
- { intent: '1', text: '7', entities: [] },
- ];
- const sampler = new ComposerBootstrapSampler(utterances, 2);
- const result = sampler.getSampledUtterances();
- const intent1 = result.filter((e) => e.intent === '1').length;
- expect(2 / intent1).toBeCloseTo(0.5, 2);
- const sampler1 = new ComposerBootstrapSampler(utterances, 5);
- const result1 = sampler1.getSampledUtterances();
- const intent11 = result1.filter((e) => e.intent === '1').length;
- expect(intent11).toBeCloseTo(5);
- });
-});
diff --git a/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts b/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts
deleted file mode 100644
index 072b0c4b9f..0000000000
--- a/Composer/packages/server/src/models/bot/sampler/__tests__/ReservoirSampler.test.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { ComposerReservoirSampler } from './../../../../models/bot/sampler/ReservoirSampler';
-
-describe('BootstrapSampler', () => {
- it('down size the number of utterances reservoir sampling', async () => {
- const utterances = [
- { intent: '0', text: '1', entities: [] },
- { intent: '1', text: '2', entities: [] },
- { intent: '1', text: '3', entities: [] },
- { intent: '1', text: '4', entities: [] },
- { intent: '1', text: '5', entities: [] },
- { intent: '1', text: '6', entities: [] },
- { intent: '1', text: '7', entities: [] },
- { intent: '1', text: '8', entities: [] },
- { intent: '1', text: '9', entities: [] },
- { intent: '1', text: '10', entities: [] },
- { intent: '1', text: '11', entities: [] },
- { intent: '1', text: '12', entities: [] },
- { intent: '1', text: '13', entities: [] },
- { intent: '1', text: '14', entities: [] },
- { intent: '1', text: '15', entities: [] },
- ];
- const sampler = new ComposerReservoirSampler(utterances, 10);
- expect(sampler.getSampledUtterances().length).toBe(10);
- const sampler1 = new ComposerReservoirSampler(utterances, 11);
- expect(sampler1.getSampledUtterances().length).toBe(11);
- const sampler2 = new ComposerReservoirSampler(utterances, 12);
- expect(sampler2.getSampledUtterances().length).toBe(12);
- const sampler3 = new ComposerReservoirSampler(utterances, 18);
- expect(sampler3.getSampledUtterances().length).toBe(15);
- });
-});
diff --git a/Composer/packages/tools/language-servers/language-understanding/package.json b/Composer/packages/tools/language-servers/language-understanding/package.json
index 0cd4ac9f1e..4bf6b692fd 100644
--- a/Composer/packages/tools/language-servers/language-understanding/package.json
+++ b/Composer/packages/tools/language-servers/language-understanding/package.json
@@ -21,7 +21,7 @@
"@bfc/indexers": "*",
"@bfc/shared": "*",
"@microsoft/bf-cli-command": "^4.10.0-preview.141651",
- "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8",
+ "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",
"express": "^4.15.2",
"monaco-languageclient": "^0.10.0",
"normalize-url": "^2.0.1",
diff --git a/Composer/plugins/azurePublish/package.json b/Composer/plugins/azurePublish/package.json
index ac443ae1c7..4b617b6d7f 100644
--- a/Composer/plugins/azurePublish/package.json
+++ b/Composer/plugins/azurePublish/package.json
@@ -21,8 +21,8 @@
"@bfc/extension": "../../packages/extension",
"@bfc/indexers": "../../packages/lib/indexers",
"@bfc/shared": "../../packages/lib/shared",
+ "@microsoft/bf-lu": "^4.11.0-dev.20201013.7ccb128",
"@botframework-composer/types": "0.0.1",
- "@microsoft/bf-lu": "^4.11.0-dev.20201005.7e5e1b8",
"@microsoft/bf-luis-cli": "^4.10.0-dev.20200721.8bb21ac",
"@types/archiver": "3.1.0",
"@types/fs-extra": "8.1.0",
diff --git a/Composer/plugins/azurePublish/yarn.lock b/Composer/plugins/azurePublish/yarn.lock
index 7af61237e8..cae01cf2cd 100644
--- a/Composer/plugins/azurePublish/yarn.lock
+++ b/Composer/plugins/azurePublish/yarn.lock
@@ -171,7 +171,7 @@
"@bfc/indexers@../../packages/lib/indexers":
version "0.0.0"
dependencies:
- "@microsoft/bf-lu" "^4.11.0-dev.20201005.7e5e1b8"
+ "@microsoft/bf-lu" "^4.11.0-dev.20201013.7ccb128"
adaptive-expressions "4.10.0-preview-147186"
botbuilder-lg "^4.10.0-preview-150886"
lodash "^4.17.19"
@@ -231,10 +231,10 @@
semver "^5.5.1"
tslib "^1.10.0"
-"@microsoft/bf-lu@^4.11.0-dev.20201005.7e5e1b8":
- version "4.11.0-dev.20201005.7e5e1b8"
- resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201005.7e5e1b8.tgz#62f9a37bb8340456e41853e179f2941dab9dfc87"
- integrity sha512-0vZ7sw+lDj0HiUa1vfaftIqhCSEMLwOl9uExR3gt1HkulakG676vtyG9Orr5z7ROTn+byhSHXdjfgFZdMtw1BQ==
+"@microsoft/bf-lu@^4.11.0-dev.20201013.7ccb128":
+ version "4.11.0-dev.20201014.d8a6b54"
+ resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201014.d8a6b54.tgz#59bdb6e94e54307f76ccadd606ca9e61b6ee4aed"
+ integrity sha512-3iTq2YSXlQaxShepP/Hu393iuyAAOsAg0IGqt88Hhm+1QhT0WJeuqxzYQeZHYitxJrpEcWfN9/PsoCkBqDEsgw==
dependencies:
"@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1"
"@azure/ms-rest-azure-js" "2.0.1"
@@ -251,7 +251,7 @@
lodash "^4.17.19"
node-fetch "~2.6.0"
semver "^5.5.1"
- tslib "^1.10.0"
+ tslib "^2.0.3"
"@microsoft/bf-luis-cli@^4.10.0-dev.20200721.8bb21ac":
version "4.10.0-dev.20200721.8bb21ac"
@@ -2524,6 +2524,11 @@ tslib@^1.10.0, tslib@^1.9.2, tslib@^1.9.3:
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043"
integrity sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==
+tslib@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c"
+ integrity sha512-uZtkfKblCEQtZKBF6EBXVZeQNl82yqtDQdv+eck8u7tdPxjLu2/lp5/uPW+um2tpuxINHWy3GhiccY7QgEaVHQ==
+
tslib@~1.10.0:
version "1.10.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a"
diff --git a/Composer/yarn.lock b/Composer/yarn.lock
index 70669267e9..a609a9ba57 100644
--- a/Composer/yarn.lock
+++ b/Composer/yarn.lock
@@ -2980,15 +2980,17 @@
fs-extra "^7.0.1"
tslib "~1.10.0"
-"@microsoft/bf-dispatcher@^4.10.0-preview.141651":
- version "4.10.0-preview.141651"
- resolved "https://botbuilder.myget.org/F/botframework-cli/npm/@microsoft/bf-dispatcher/-/@microsoft/bf-dispatcher-4.10.0-preview.141651.tgz#d943197c42e15894a55eadf4c5e3d85d9bbccfd7"
- integrity sha1-2UMZfELhWJSlXq30xePYXZu8z9c=
+"@microsoft/bf-dispatcher@^4.11.0-beta.20201015.008a3a4":
+ version "4.11.0-beta.20201015.008a3a4"
+ resolved "https://registry.yarnpkg.com/@microsoft/bf-dispatcher/-/bf-dispatcher-4.11.0-beta.20201015.008a3a4.tgz#a80b68b8a123653ae52e35cde25cb2d1d1b98390"
+ integrity sha512-EN1hdAKSh8siN4UDuhMsM5QJtVlAj5X6peiXB8z18rPNL7oyGUFzgEfUoQ9ZqfWpRUasz1Zj8sAyoFS76NpkGQ==
dependencies:
- "@microsoft/bf-lu" "4.10.0-preview.141651"
+ "@microsoft/bf-lu" next
"@oclif/command" "~1.5.19"
"@oclif/config" "~1.13.3"
argparse "~1.0.10"
+ readline-sync "^1.4.10"
+ ts-md5 "^1.2.6"
tslib "^1.10.0"
"@microsoft/bf-generate-library@^4.10.0-daily.20201008.172736":
@@ -3009,15 +3011,13 @@
seedrandom "~3.0.5"
swagger-parser "^8.0.4"
-"@microsoft/bf-lu@4.10.0-preview.141651":
- version "4.10.0-preview.141651"
- resolved "https://botbuilder.myget.org/F/botframework-cli/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.10.0-preview.141651.tgz#29ed2af803d23ee760354913f5b814873bc1285c"
- integrity sha1-Ke0q+APSPudgNUkT9bgUhzvBKFw=
+"@microsoft/bf-lu@^4.11.0-dev.20200831.d7b6149":
+ version "4.11.0-dev.20201008.18fba61"
+ resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201008.18fba61.tgz#7ffcb888309f06d518a85d342c62dc4b843d2949"
+ integrity sha1-f/y4iDCfBtUYqF00LGLcS4Q9KUk=
dependencies:
"@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1"
"@azure/ms-rest-azure-js" "2.0.1"
- "@oclif/command" "~1.5.19"
- "@oclif/errors" "~1.2.2"
"@types/node-fetch" "~2.5.5"
antlr4 "^4.7.2"
chalk "2.4.1"
@@ -3028,15 +3028,15 @@
get-stdin "^6.0.0"
globby "^10.0.1"
intercept-stdout "^0.1.2"
- lodash "^4.17.15"
+ lodash "^4.17.19"
node-fetch "~2.6.0"
semver "^5.5.1"
tslib "^1.10.0"
-"@microsoft/bf-lu@^4.11.0-dev.20200831.d7b6149":
- version "4.11.0-dev.20201008.18fba61"
- resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201008.18fba61.tgz#7ffcb888309f06d518a85d342c62dc4b843d2949"
- integrity sha1-f/y4iDCfBtUYqF00LGLcS4Q9KUk=
+"@microsoft/bf-lu@^4.11.0-dev.20201013.7ccb128":
+ version "4.11.0-dev.20201013.7ccb128"
+ resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201013.7ccb128.tgz#9dbb5043d3f7a384d1449c73fd984016b5115ca4"
+ integrity sha512-xaG5yDxtdwNNfYa6cs9wmCGTN6K6d2sh8jfGvMt7dqs26My3c6Sus03VF74sRrp2OQXnbBLqZPVY1qzLKnjoJg==
dependencies:
"@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1"
"@azure/ms-rest-azure-js" "2.0.1"
@@ -3053,12 +3053,12 @@
lodash "^4.17.19"
node-fetch "~2.6.0"
semver "^5.5.1"
- tslib "^1.10.0"
+ tslib "^2.0.3"
-"@microsoft/bf-lu@^4.11.0-dev.20201005.7e5e1b8":
- version "4.11.0-dev.20201005.7e5e1b8"
- resolved "https://botbuilder.myget.org/F/botbuilder-declarative/npm/@microsoft/bf-lu/-/@microsoft/bf-lu-4.11.0-dev.20201005.7e5e1b8.tgz#62f9a37bb8340456e41853e179f2941dab9dfc87"
- integrity sha1-Yvmje7g0BFbkGFPhefKUHaud/Ic=
+"@microsoft/bf-lu@next":
+ version "4.11.0-dev.20201015.a41c691"
+ resolved "https://registry.yarnpkg.com/@microsoft/bf-lu/-/bf-lu-4.11.0-dev.20201015.a41c691.tgz#1002eb10b7625fead68274c007ac857de79f4446"
+ integrity sha512-Bq/4NJ8FpJV/wOOdjxLPMHmZoT08qSKozFbnukjdh5L0UNDDDyhTSSLmG1hdGc337VmtQF6YBsLu9sEmATSJUA==
dependencies:
"@azure/cognitiveservices-luis-authoring" "4.0.0-preview.1"
"@azure/ms-rest-azure-js" "2.0.1"
@@ -3075,7 +3075,7 @@
lodash "^4.17.19"
node-fetch "~2.6.0"
semver "^5.5.1"
- tslib "^1.10.0"
+ tslib "^2.0.3"
"@microsoft/load-themed-styles@^1.10.26":
version "1.10.39"
@@ -16460,6 +16460,11 @@ readdirp@~3.4.0:
dependencies:
picomatch "^2.2.1"
+readline-sync@^1.4.10:
+ version "1.4.10"
+ resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
+ integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==
+
readline-sync@^1.4.9:
version "1.4.9"
resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.9.tgz#3eda8e65f23cd2a17e61301b1f0003396af5ecda"
@@ -18636,7 +18641,7 @@ ts-loader@^6.0.1:
micromatch "^4.0.0"
semver "^6.0.0"
-ts-md5@^1.2.7:
+ts-md5@^1.2.6, ts-md5@^1.2.7:
version "1.2.7"
resolved "https://registry.yarnpkg.com/ts-md5/-/ts-md5-1.2.7.tgz#b76471fc2fd38f0502441f6c3b9494ed04537401"
integrity sha512-emODogvKGWi1KO1l9c6YxLMBn6CEH3VrH5mVPIyOtxBG52BvV4jP3GWz6bOZCz61nLgBc3ffQYE4+EHfCD+V7w==
@@ -18713,6 +18718,11 @@ tslib@^2.0.0:
resolved "https://botbuilder.myget.org/F/botframework-cli/npm/tslib/-/tslib-2.0.0.tgz#18d13fc2dce04051e20f074cc8387fd8089ce4f3"
integrity sha1-GNE/wtzgQFHiDwdMyDh/2Aic5PM=
+tslib@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c"
+ integrity sha512-uZtkfKblCEQtZKBF6EBXVZeQNl82yqtDQdv+eck8u7tdPxjLu2/lp5/uPW+um2tpuxINHWy3GhiccY7QgEaVHQ==
+
tsutils@^3.17.1:
version "3.17.1"
resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759"