From 1d1bdc23e6f238497fbd15b148976b4e28fd67d3 Mon Sep 17 00:00:00 2001 From: leilzh Date: Thu, 26 Nov 2020 18:05:52 +0800 Subject: [PATCH 1/8] refactor the luis build in azure publish --- .../src/recoilModel/dispatchers/publisher.ts | 22 +- .../server/src/models/bot/botProject.ts | 1 + .../packages/server/src/models/bot/builder.ts | 34 +- extensions/azurePublish/package.json | 2 - extensions/azurePublish/src/deploy.ts | 13 +- extensions/azurePublish/src/index.ts | 24 +- extensions/azurePublish/src/luisAndQnA.ts | 426 ++++-------------- .../azurePublish/src/utils/crossTrainUtil.ts | 216 --------- extensions/azurePublish/src/utils/fileUtil.ts | 12 - extensions/azurePublish/src/utils/jsonWalk.ts | 37 -- 10 files changed, 150 insertions(+), 637 deletions(-) delete mode 100644 extensions/azurePublish/src/utils/crossTrainUtil.ts delete mode 100644 extensions/azurePublish/src/utils/fileUtil.ts delete mode 100644 extensions/azurePublish/src/utils/jsonWalk.ts diff --git a/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts b/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts index 96f342d7c7..7cfe771ba3 100644 --- a/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts +++ b/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts @@ -13,8 +13,12 @@ import { botLoadErrorState, isEjectRuntimeExistState, filePersistenceState, + luFilesState, + qnaFilesState, } from '../atoms/botState'; import { botEndpointsState } from '../atoms'; +import { dialogsSelectorFamily } from '../selectors'; +import * as luUtil from '../../utils/luUtil'; import { BotStatus, Text } from './../../constants'; import httpClient from './../../utils/httpUtil'; @@ -121,10 +125,24 @@ export const publisherDispatcher = () => { }); const publishToTarget = useRecoilCallback( - (callbackHelpers: CallbackInterface) => async (projectId: string, target: any, metadata, sensitiveSettings) => { + (callbackHelpers: CallbackInterface) => async ( + projectId: string, + target: any, + metadata: any, + sensitiveSettings + ) => { try { + const { snapshot } = callbackHelpers; + const dialogs = await snapshot.getPromise(dialogsSelectorFamily(projectId)); + const luFiles = await snapshot.getPromise(luFilesState(projectId)); + const qnaFiles = await snapshot.getPromise(qnaFilesState(projectId)); + const referredLuFiles = luUtil.checkLuisBuild(luFiles, dialogs); const response = await httpClient.post(`/publish/${projectId}/publish/${target.name}`, { - metadata, + metadata: { + ...metadata, + luResources: referredLuFiles.map((file) => file.id), + qnaResources: qnaFiles.map((file) => file.id), + }, sensitiveSettings, }); await publishSuccess(callbackHelpers, projectId, response.data, target); diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index b7141f24c7..f189698e3c 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -494,6 +494,7 @@ export class BotProject implements IBotProject { } }); + this.builder.rootDir = this.dir; this.builder.setBuildConfig( { ...luisConfig, subscriptionKey: qnaConfig.subscriptionKey, qnaRegion: qnaConfig.qnaRegion }, this.settings.downsampling diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index bd11751898..61f1c8df9a 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. /* eslint-disable @typescript-eslint/no-var-requires */ -import { pathExists, writeFile } from 'fs-extra'; +import { pathExists, writeFile, copy } from 'fs-extra'; import { FileInfo, IConfig, SDKKinds } from '@bfc/shared'; import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler'; import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler'; @@ -51,6 +51,7 @@ export class Builder { public config: IConfig | null = null; public downSamplingConfig: DownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 }; private _locale: string; + private containOrchestrator = false; private luBuilder = new luBuild.Builder((message) => { log(message); @@ -67,6 +68,12 @@ export class Builder { this._locale = locale; } + public set rootDir(path: string) { + this.botDir = path; + this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER); + this.interruptionFolderPath = Path.join(this.generatedFolderPath, INTERRUPTION); + } + public build = async (luFiles: FileInfo[], qnaFiles: FileInfo[], allFiles: FileInfo[]) => { try { await this.createGeneratedDir(); @@ -76,7 +83,11 @@ export class Builder { const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles(); const { luBuildFiles, orchestratorBuildFiles } = this.separateLuFiles(interruptionLuFiles, allFiles); - + if (orchestratorBuildFiles.length) { + this.containOrchestrator = true; + } else { + this.containOrchestrator = false; + } await this.runLuBuild(luBuildFiles); await this.runQnaBuild(interruptionQnaFiles); await this.runOrchestratorBuild(orchestratorBuildFiles); @@ -219,6 +230,25 @@ export class Builder { return await Orchestrator.buildAsync(modelPath, luObjects, isDialog, null, fullEmbedding); } + public async copyModelPathToBot() { + if (this.containOrchestrator) { + const nlrList = await this.runOrchestratorNlrList(); + const defaultNLR = nlrList.default; + const folderName = defaultNLR.replace('.onnx', ''); + const modelPath = Path.resolve(await this.getModelPathAsync(), folderName); + const destDir = Path.resolve(this.generatedFolderPath, folderName); + await copy(modelPath, destDir); + await this.updateOrchestratorSetting(folderName); + } + } + + private async updateOrchestratorSetting(dirName: string) { + const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json'); + const content = JSON.parse(await this.storage.readFile(settingPath)); + content.orchestrator.ModelPath = './ComposerDialogs/generated/' + dirName; + await this.storage.writeFile(settingPath, JSON.stringify(content, null, 2)); + } + private async createGeneratedDir() { // clear previous folder await this.deleteDir(this.generatedFolderPath); diff --git a/extensions/azurePublish/package.json b/extensions/azurePublish/package.json index 560354c35f..7e5da1f2e1 100644 --- a/extensions/azurePublish/package.json +++ b/extensions/azurePublish/package.json @@ -20,8 +20,6 @@ "@azure/ms-rest-nodeauth": "3.0.3", "@bfc/indexers": "../../Composer/packages/lib/indexers", "@bfc/shared": "../../Composer/packages/lib/shared", - "@microsoft/bf-lu": "4.11.0-dev.20201005.7e5e1b8", - "@microsoft/bf-luis-cli": "^4.10.0-dev.20200721.8bb21ac", "adal-node": "0.2.1", "archiver": "^5.0.2", "fs-extra": "8.1.0", diff --git a/extensions/azurePublish/src/deploy.ts b/extensions/azurePublish/src/deploy.ts index 0e26915d6d..a5224c2c1f 100644 --- a/extensions/azurePublish/src/deploy.ts +++ b/extensions/azurePublish/src/deploy.ts @@ -8,7 +8,7 @@ import * as rp from 'request-promise'; import { BotProjectDeployConfig } from './botProjectDeployConfig'; import { BotProjectDeployLoggerType } from './botProjectLoggerType'; -import { LuisAndQnaPublish } from './luisAndQnA'; +import { build, publishLuisToPrediction } from './luisAndQnA'; import archiver = require('archiver'); export class BotProjectDeploy { @@ -58,18 +58,19 @@ export class BotProjectDeploy { if (!language) { language = 'en-us'; } - const publisher = new LuisAndQnaPublish({ logger: this.logger, projPath: this.projPath }); + + build(project, this.projPath, settings); // this function returns an object that contains the luis APP ids mapping // each dialog to its matching app. - const { luisAppIds, qnaConfig } = await publisher.publishLuisAndQna( + const { luisAppIds, qnaConfig } = await publishLuisToPrediction( name, environment, this.accessToken, - language, settings.luis, - settings.qna, - luisResource + luisResource, + this.projPath, + this.logger ); // amend luis settings with newly generated values diff --git a/extensions/azurePublish/src/index.ts b/extensions/azurePublish/src/index.ts index 7c99e6cce1..36f2059718 100644 --- a/extensions/azurePublish/src/index.ts +++ b/extensions/azurePublish/src/index.ts @@ -126,7 +126,8 @@ export default async (composer: IExtensionRegistration): Promise => { */ private init = async (project: any, srcTemplate: string, resourcekey: string, runtime: any) => { // point to the declarative assets (possibly in remote storage) - const botFiles = project.getProject().filesWithoutRecognizers; + const botFiles = project.getProject().files; + console.log(botFiles); const botFolder = this.getBotFolder(resourcekey, this.mode); const runtimeFolder = this.getRuntimeFolder(resourcekey); @@ -165,13 +166,13 @@ export default async (composer: IExtensionRegistration): Promise => { * @param resourcekey */ private async cleanup(resourcekey: string) { - try { - const projFolder = this.getRuntimeFolder(resourcekey); - await emptyDir(projFolder); - await rmdir(projFolder); - } catch (error) { - this.logger('$O', error); - } + // try { + // const projFolder = this.getRuntimeFolder(resourcekey); + // await emptyDir(projFolder); + // await rmdir(projFolder); + // } catch (error) { + // this.logger('$O', error); + // } } /** @@ -321,6 +322,8 @@ export default async (composer: IExtensionRegistration): Promise => { defaultLanguage, settings, accessToken, + luResources, + qnaResources } = config; // get the appropriate runtime template which contains methods to build and configure the runtime @@ -347,7 +350,7 @@ export default async (composer: IExtensionRegistration): Promise => { // Merge all the settings // this combines the bot-wide settings, the environment specific settings, and 2 new fields needed for deployed bots // these will be written to the appropriate settings file inside the appropriate runtime plugin. - const mergedSettings = mergeDeep(fullSettings, settings); + const mergedSettings = mergeDeep(fullSettings, settings, {luResources, qnaResources}); // Prepare parameters and then perform the actual deployment action const customizeConfiguration: CreateAndDeployResources = { @@ -392,6 +395,7 @@ export default async (composer: IExtensionRegistration): Promise => { * plugin methods *************************************************************************************************/ publish = async (config: PublishConfig, project: IBotProject, metadata, user) => { + const {luResources, qnaResources} = metadata; const { // these are provided by Composer profileName, // the name of the publishing profile "My Azure Prod Slot" @@ -436,7 +440,7 @@ export default async (composer: IExtensionRegistration): Promise => { throw new Error('Required field `settings` is missing from publishing profile.'); } - this.asyncPublish(config, project, resourcekey, jobId); + this.asyncPublish({...config, luResources, qnaResources}, project, resourcekey, jobId); } catch (err) { this.logger('%O', err); if (err instanceof Error) { diff --git a/extensions/azurePublish/src/luisAndQnA.ts b/extensions/azurePublish/src/luisAndQnA.ts index 474c763713..140ee200da 100644 --- a/extensions/azurePublish/src/luisAndQnA.ts +++ b/extensions/azurePublish/src/luisAndQnA.ts @@ -6,182 +6,68 @@ import { promisify } from 'util'; import * as fs from 'fs-extra'; import * as rp from 'request-promise'; -import { ILuisConfig, FileInfo, IQnAConfig } from '@botframework-composer/types'; +import { ILuisConfig, FileInfo, IQnAConfig, IBotProject } from '@botframework-composer/types'; -import { ICrossTrainConfig, createCrossTrainConfig } from './utils/crossTrainUtil'; import { BotProjectDeployLoggerType } from './botProjectLoggerType'; -import { luImportResolverGenerator } from '@bfc/shared/lib/luBuildResolver'; -const crossTrainer = require('@microsoft/bf-lu/lib/parser/cross-train/crossTrainer.js'); -const luBuild = require('@microsoft/bf-lu/lib/parser/lubuild/builder.js'); -const qnaBuild = require('@microsoft/bf-lu/lib/parser/qnabuild/builder.js'); const readdir: any = promisify(fs.readdir); -export interface PublishConfig { - // Logger - logger: (string) => any; - projPath: string; - [key: string]: any; -} - -const INTERRUPTION = 'interruption'; - -export class LuisAndQnaPublish { - private logger: (string) => any; - private remoteBotPath: string; - private generatedFolder: string; - private interruptionFolderPath: string; - private crossTrainConfig: ICrossTrainConfig; - - constructor(config: PublishConfig) { - this.logger = config.logger; - // path to the ready to deploy generated folder - this.remoteBotPath = path.join(config.projPath, 'ComposerDialogs'); - this.generatedFolder = path.join(this.remoteBotPath, 'generated'); - this.interruptionFolderPath = path.join(this.generatedFolder, INTERRUPTION); - - // Cross Train config - this.crossTrainConfig = { - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - botName: '', - }; - } +const botPath = (projPath: string) => path.join(projPath, 'ComposerDialogs') - /*******************************************************************************************************************************/ - /* This section has to do with publishing LU files to LUIS - /*******************************************************************************************************************************/ +type QnaConfigType = { + subscriptionKey: string; + qnaRegion: string | 'westus'; +}; - /** - * return an array of all the files in a given directory - * @param dir - */ - private async getFiles(dir: string): Promise { - const dirents = await readdir(dir, { withFileTypes: true }); - const files = await Promise.all( - dirents.map((dirent) => { - const res = path.resolve(dir, dirent.name); - return dirent.isDirectory() ? this.getFiles(res) : res; - }) - ); - return Array.prototype.concat(...files); - } - - /** - * Helper function to get the appropriate account out of a list of accounts - * @param accounts - * @param filter - */ - private getAccount(accounts: any, filter: string) { - for (const account of accounts) { - if (account.AccountName === filter) { - return account; - } - } - } +type Resources = { + luResources: string[]; + qnaResources: string[]; +} - private notEmptyModel(file: string) { - return fs.readFileSync(file).length > 0; - } +type BuildSettingType = { + luis: ILuisConfig, + qna: QnaConfigType +} & Resources - private async createGeneratedDir() { - if (!(await fs.pathExists(this.generatedFolder))) { - await fs.mkdir(this.generatedFolder); +function getAccount(accounts: any, filter: string) { + for (const account of accounts) { + if (account.AccountName === filter) { + return account; } } +} - private async setCrossTrainConfig(botName: string, dialogFiles: string[], luFiles: string[]) { - const dialogs: { [key: string]: any }[] = []; - for (const dialog of dialogFiles) { - dialogs.push({ - id: dialog.substring(dialog.lastIndexOf('\\') + 1, dialog.length), - isRoot: dialog.indexOf(path.join(this.remoteBotPath, 'dialogs')) === -1, - content: fs.readJSONSync(dialog), - }); - } - const luFileInfos: FileInfo[] = luFiles.map((luFile) => { - const fileStats = fs.statSync(luFile); - return { - name: luFile.substring(luFile.lastIndexOf('\\') + 1), - content: fs.readFileSync(luFile, 'utf-8'), - lastModified: fileStats.mtime.toString(), - path: luFile, - relativePath: luFile.substring(luFile.lastIndexOf(this.remoteBotPath) + 1), - }; - }); - this.crossTrainConfig = createCrossTrainConfig(dialogs, luFileInfos); - } - private async writeCrossTrainFiles(crossTrainResult) { - if (!(await fs.pathExists(this.interruptionFolderPath))) { - await fs.mkdir(this.interruptionFolderPath); - } +/** +* return an array of all the files in a given directory +* @param dir +*/ +async function getFiles(dir: string): Promise { + const dirents = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + dirents.map((dirent) => { + const res = path.resolve(dir, dirent.name); + return dirent.isDirectory() ? getFiles(res) : res; + }) + ); + return Array.prototype.concat(...files); +} - await Promise.all( - [...crossTrainResult.keys()].map(async (key: string) => { - const fileName = path.basename(key); - const newFileId = path.join(this.interruptionFolderPath, fileName); - await fs.writeFile(newFileId, crossTrainResult.get(key).Content); - }) - ); - } +export async function publishLuisToPrediction( + name: string, + environment: string, + accessToken: string, + luisSettings: ILuisConfig, + luisResource: string, + path: string, + logger + ) { + let { authoringKey: luisAuthoringKey, endpoint: luisEndpoint, authoringRegion: luisAuthoringRegion } = luisSettings; - private async crossTrain(luFiles: string[], qnaFiles: string[]) { - const luContents: { [key: string]: any }[] = []; - const qnaContents: { [key: string]: any }[] = []; - for (const luFile of luFiles) { - luContents.push({ - content: fs.readFileSync(luFile, { encoding: 'utf-8' }), - name: path.basename(luFile), - id: path.basename(luFile), - path: luFile, - }); - } - for (const qnaFile of qnaFiles) { - qnaContents.push({ - content: fs.readFileSync(qnaFile, { encoding: 'utf-8' }), - name: path.basename(qnaFile), - id: path.basename(qnaFile), - path: qnaFile, - }); + if (!luisSettings.endpoint) { + luisEndpoint = `https://${luisAuthoringRegion}.api.cognitive.microsoft.com`; } - const importResolver = luImportResolverGenerator([...qnaContents, ...luContents] as FileInfo[]); - const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, importResolver); - await this.writeCrossTrainFiles(result.luResult); - await this.writeCrossTrainFiles(result.qnaResult); - } - - private async cleanCrossTrain() { - fs.rmdirSync(this.interruptionFolderPath, { recursive: true }); - } - private async getInterruptionFiles() { - const files = await this.getFiles(this.interruptionFolderPath); - const interruptionLuFiles: string[] = []; - const interruptionQnaFiles: string[] = []; - files.forEach((file) => { - if (file.endsWith('qna')) { - interruptionQnaFiles.push(file); - } else if (file.endsWith('lu')) { - interruptionLuFiles.push(file); - } - }); - return { interruptionLuFiles, interruptionQnaFiles }; - } - - private async publishLuis( - name: string, - environment: string, - accessToken: string, - language: string, - luisSettings: ILuisConfig, - interruptionLuFiles: string[], - luisResource?: string - ) { - const { authoringKey: luisAuthoringKey, endpoint: luisEndpoint } = luisSettings; - - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'start publish luis', }); @@ -189,7 +75,7 @@ export class LuisAndQnaPublish { // Find any files that contain the name 'luis.settings' in them // These are generated by the LuBuild process and placed in the generated folder // These contain dialog-to-luis app id mapping - const luisConfigFiles = (await this.getFiles(this.remoteBotPath)).filter((filename) => + const luisConfigFiles = (await getFiles(botPath(path))).filter((filename) => filename.includes('luis.settings') ); const luisAppIds: any = {}; @@ -208,14 +94,17 @@ export class LuisAndQnaPublish { // DOCS HERE: https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5be313cec181ae720aa2b26c // This returns a list of azure account information objects with AzureSubscriptionID, ResourceGroup, AccountName for each. const getAccountUri = `${luisEndpoint}/luis/api/v2.0/azureaccounts`; + console.log(getAccountUri); const options = { headers: { Authorization: `Bearer ${accessToken}`, 'Ocp-Apim-Subscription-Key': luisAuthoringKey }, } as rp.RequestPromiseOptions; const response = await rp.get(getAccountUri, options); // this should include an array of account info objects + console.log(response); accountList = JSON.parse(response); } catch (err) { + console.log(err); // handle the token invalid const error = JSON.parse(err.error); if (error?.error?.message && error?.error?.message.indexOf('access token expiry') > 0) { @@ -228,13 +117,13 @@ export class LuisAndQnaPublish { } // Extract the accoutn object that matches the expected resource name. // This is the name that would appear in the azure portal associated with the luis endpoint key. - const account = this.getAccount(accountList, luisResource ? luisResource : `${name}-${environment}-luis`); + const account = getAccount(accountList, luisResource ? luisResource : `${name}-${environment}-luis`); // Assign the appropriate account to each of the applicable LUIS apps for this bot. // DOCS HERE: https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5be32228e8473de116325515 for (const dialogKey in luisAppIds) { const luisAppId = luisAppIds[dialogKey].appId; - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: `Assigning to luis app id: ${luisAppId}`, }); @@ -249,209 +138,46 @@ export class LuisAndQnaPublish { // TODO: Add some error handling on this API call. As it is, errors will just throw by default and be caught by the catch all try/catch in the deploy method - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: response, }); } // The process has now completed. - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'Luis Publish Success! ...', }); // return the new settings that need to be added to the main settings file. return luisAppIds; - } - - // Run through the lubuild process - // This happens in the build folder, NOT in the original source folder - private async buildLuis( - name: string, - environment: string, - language: string, - luisSettings: ILuisConfig, - interruptionLuFiles: string[] - ) { - const { authoringKey: luisAuthoringKey, authoringRegion: luisAuthoringRegion } = luisSettings; - - // Instantiate the LuBuild object from the LU parsing library - // This object is responsible for parsing the LU files and sending them to LUIS - const builder = new luBuild.Builder((msg) => - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: msg, - }) - ); - - // Pass in the list of the non-empty LU files we got above... - const loadResult = await builder.loadContents( - interruptionLuFiles, - language || 'en-us', - environment || '', - luisAuthoringRegion || '' - ); - - // set the default endpoint - if (!luisSettings.endpoint) { - luisSettings.endpoint = `https://${luisAuthoringRegion}.api.cognitive.microsoft.com`; - } - - // if not specified, set the authoring endpoint - if (!luisSettings.authoringEndpoint) { - luisSettings.authoringEndpoint = luisSettings.endpoint; - } - - // Perform the Lubuild process - // This will create new luis apps for each of the luis models represented in the LU files - const buildResult = await builder.build( - loadResult.luContents, - loadResult.recognizers, - luisAuthoringKey, - luisSettings.authoringEndpoint, - name, - environment, - language, - true, - false, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - - // Write the generated files to the generated folder - await builder.writeDialogAssets(buildResult, true, this.generatedFolder); - - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: `lubuild succeed`, - }); - } - - private async buildQna( - name: string, - environment: string, - language: string, - qnaSettings: IQnAConfig, - interruptionQnaFiles: string[] - ) { - // eslint-disable-next-line prefer-const - let { subscriptionKey } = qnaSettings; - const authoringRegion = 'westus'; - // publishing luis - const builder = new qnaBuild.Builder((msg) => - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: msg, - }) - ); - - const loadResult = await builder.loadContents( - interruptionQnaFiles, - name, - environment || '', - authoringRegion || '', - language || '' - ); - - const endpoint = `https://${authoringRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; - - const buildResult = await builder.build( - loadResult.qnaContents, - loadResult.recognizers, - subscriptionKey, - endpoint, - name, - environment, - language, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - await builder.writeDialogAssets(buildResult, true, this.generatedFolder); - - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: `qnabuild succeed`, - }); - - // Find any files that contain the name 'qnamaker.settings' in them - // These are generated by the LuBuild process and placed in the generated folder - // These contain dialog-to-luis app id mapping - const qnaConfigFile = (await this.getFiles(this.remoteBotPath)).find((filename) => - filename.includes('qnamaker.settings') - ); - const qna: any = {}; - - // Read the qna settings - if (qnaConfigFile) { - const qnaConfig = await fs.readJson(qnaConfigFile); - const endpointKey = await builder.getEndpointKeys(subscriptionKey, endpoint); - Object.assign(qna, qnaConfig.qna, { endpointKey: endpointKey.primaryEndpointKey }); - } - return qna; - } +} - // Run through the build process - // This happens in the build folder, NOT in the original source folder - public async publishLuisAndQna( - name: string, - environment: string, - accessToken: string, - language: string, - luisSettings: ILuisConfig, - qnaSettings: IQnAConfig, - luisResource?: string - ) { - const { authoringKey, authoringRegion } = luisSettings; - const { subscriptionKey } = qnaSettings; - const botFiles = await this.getFiles(this.remoteBotPath); - const luFiles = botFiles.filter((name) => { - return name.endsWith('.lu'); - }); - const qnaFiles = botFiles.filter((name) => { - return name.endsWith('.qna'); - }); +export async function build(project: IBotProject, path: string, settings: BuildSettingType) { + const {luResources, qnaResources, luis: luisConfig, qna: qnaConfig} = settings; - // check content - const notEmptyLuFiles = luFiles.some((name) => this.notEmptyModel(name)); - const notEmptyQnaFiles = qnaFiles.some((name) => this.notEmptyModel(name)); + const {builder, files} = project; - if (notEmptyLuFiles && !(authoringKey && authoringRegion)) { - throw Error('Should have luis authoringKey and authoringRegion when lu file not empty'); + const luFiles: FileInfo[] = []; + luResources.forEach((id) => { + const fileName = `${id}.lu`; + const f = files.get(fileName); + if (f) { + luFiles.push(f); } - if (notEmptyQnaFiles && !subscriptionKey) { - throw Error('Should have qna subscriptionKey when qna file not empty'); + }); + const qnaFiles: FileInfo[] = []; + qnaResources.forEach((id) => { + const fileName = `${id}.qna`; + const f = files.get(fileName); + if (f) { + qnaFiles.push(f); } - const dialogFiles = botFiles.filter((name) => { - return name.endsWith('.dialog') && this.notEmptyModel(name); - }); - - await this.setCrossTrainConfig(name, dialogFiles, luFiles); - await this.createGeneratedDir(); - await this.crossTrain(luFiles, qnaFiles); - const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles(); + }); - await this.buildLuis(name, environment, language, luisSettings, interruptionLuFiles); - let luisAppIds = {}; - // publish luis only when Lu files not empty - if (notEmptyLuFiles) { - luisAppIds = await this.publishLuis( - name, - environment, - accessToken, - language, - luisSettings, - interruptionLuFiles, - luisResource - ); - } - - const qnaConfig = await this.buildQna(name, environment, language, qnaSettings, interruptionQnaFiles); - await this.cleanCrossTrain(); - return { luisAppIds, qnaConfig }; - } + builder.rootDir = botPath(path); + builder.setBuildConfig( {...luisConfig, ...qnaConfig}, project.settings.downsampling ); + await builder.build(luFiles, qnaFiles, Array.from(files.values()) as FileInfo[]); + await builder.copyModelPathToBot(); } diff --git a/extensions/azurePublish/src/utils/crossTrainUtil.ts b/extensions/azurePublish/src/utils/crossTrainUtil.ts deleted file mode 100644 index 5dad90688c..0000000000 --- a/extensions/azurePublish/src/utils/crossTrainUtil.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * luUtil.ts is a single place use lu-parser handle lu file operation. - * it's designed have no state, input text file, output text file. - * for more usage detail, please check client/__tests__/utils/luUtil.test.ts - */ -import keys from 'lodash/keys'; -import { FieldNames } from '@bfc/shared'; -import { LuFile, DialogInfo, IIntentTrigger, SDKKinds, FileInfo } from '@botframework-composer/types'; -import { luIndexer } from '@bfc/indexers'; - -import { getBaseName, getExtension } from './fileUtil'; -import { VisitorFunc, JsonWalk } from './jsonWalk'; - -export function getReferredLuFiles(luFiles: LuFile[], dialogs: DialogInfo[], checkContent = true) { - return luFiles.filter((file) => { - const idWithOutLocale = getBaseName(file.id); - return dialogs.some( - (dialog) => dialog.luFile === idWithOutLocale && ((checkContent && !!file.content) || !checkContent) - ); - }); -} - -function ExtractAllBeginDialogs(value: any): string[] { - const dialogs: string[] = []; - - const visitor: VisitorFunc = (path: string, value: any): boolean => { - if (value?.$kind === SDKKinds.BeginDialog && value?.dialog) { - dialogs.push(value.dialog); - return true; - } - return false; - }; - - JsonWalk('$', value, visitor); - - return dialogs; -} - -// find out all properties from given dialog -function ExtractIntentTriggers(value: any): IIntentTrigger[] { - const intentTriggers: IIntentTrigger[] = []; - const triggers = value?.[FieldNames.Events]; - - if (triggers && triggers.length) { - for (const trigger of triggers) { - const dialogs = ExtractAllBeginDialogs(trigger); - - if (trigger.$kind === SDKKinds.OnIntent && trigger.intent) { - intentTriggers.push({ intent: trigger.intent, dialogs }); - } else if (trigger.$kind !== SDKKinds.OnIntent && dialogs.length) { - const emptyIntent = intentTriggers.find((e) => e.intent === ''); - if (emptyIntent) { - //remove the duplication dialogs - const all = new Set([...emptyIntent.dialogs, ...dialogs]); - emptyIntent.dialogs = Array.from(all); - } else { - intentTriggers.push({ intent: '', dialogs }); - } - } - } - } - - return intentTriggers; -} - -function createConfigId(fileId) { - return `${fileId}.lu`; -} - -function getLuFilesByDialogId(dialogId: string, luFiles: LuFile[]) { - return luFiles.filter((lu) => getBaseName(lu.id) === dialogId).map((lu) => createConfigId(lu.id)); -} - -function getFileLocale(fileName: string) { - //file name = 'a.en-us.lu' - return getExtension(getBaseName(fileName)); -} - -//replace the dialogId with luFile's name -function addLocaleToConfig(config: ICrossTrainConfig, luFiles: LuFile[]) { - const { rootIds, triggerRules } = config; - config.rootIds = rootIds.reduce((result: string[], id: string) => { - return [...result, ...getLuFilesByDialogId(id, luFiles)]; - }, []); - config.triggerRules = keys(triggerRules).reduce((result, key) => { - const fileNames = getLuFilesByDialogId(key, luFiles); - return { - ...result, - ...fileNames.reduce((result, name) => { - const locale = getFileLocale(name); - const triggers = triggerRules[key]; - keys(triggers).forEach((trigger) => { - if (!result[name]) result[name] = {}; - const ids = triggers[trigger]; - if (Array.isArray(ids)) { - result[name][trigger] = ids.map((id) => (id ? `${id}.${locale}.lu` : id)); - } else { - result[name][trigger] = ids ? `${ids}.${locale}.lu` : ids; - } - }); - return result; - }, {}), - }; - }, {}); - return config; -} - -function parse(dialog) { - const { id, content, isRoot } = dialog; - const luFile = typeof content.recognizer === 'string' ? getBaseName(id) : ''; - const qnaFile = typeof content.recognizer === 'string' ? getBaseName(id) : ''; - - return { - id: getBaseName(id), - isRoot: isRoot, - content, - luFile: luFile, - qnaFile: qnaFile, - intentTriggers: ExtractIntentTriggers(content), - }; -} -export interface ICrossTrainConfig { - rootIds: string[]; - triggerRules: { [key: string]: any }; - intentName: string; - verbose: boolean; - botName: string; -} - -//generate the cross-train config without locale -/* the config is like - { - rootIds: [ - 'main.en-us.lu', - 'main.fr-fr.lu' - ], - triggerRules: { - 'main.en-us.lu': { - 'dia1_trigger': 'dia1.en-us.lu', - 'dia2_trigger': 'dia2.en-us.lu' - }, - 'dia2.en-us.lu': { - 'dia3_trigger': 'dia3.en-us.lu', - 'dia4_trigger': 'dia4.en-us.lu' - }, - 'main.fr-fr.lu': { - 'dia1_trigger': 'dia1.fr-fr.lu' - } - }, - intentName: '_Interruption', - verbose: true - } - */ -export function createCrossTrainConfig(dialogs: any[], luFilesInfo: FileInfo[], luFeatures = {}): ICrossTrainConfig { - const triggerRules = {}; - const countMap = {}; - const wrapDialogs: { [key: string]: any }[] = []; - for (const dialog of dialogs) { - wrapDialogs.push(parse(dialog)); - } - - const luFiles = luIndexer.index(luFilesInfo, luFeatures); - - //map all referred lu files - luFiles.forEach((file) => { - countMap[getBaseName(file.id)] = 1; - }); - - let rootId = ''; - let botName = ''; - wrapDialogs.forEach((dialog) => { - if (dialog.isRoot) { - rootId = dialog.id; - botName = dialog.content.$designer.name; - } - - if (luFiles.find((luFile) => getBaseName(luFile.id) === dialog.luFile)) { - const { intentTriggers } = dialog; - const fileId = dialog.id; - //find the trigger's dialog that use a recognizer - intentTriggers.forEach((item) => { - //find all dialogs in trigger that has a luis recognizer - const used = item.dialogs.filter((dialog) => !!countMap[dialog]); - - const deduped = Array.from(new Set(used)); - - const result = {}; - if (deduped.length === 1) { - result[item.intent] = deduped[0]; - } else if (deduped.length) { - result[item.intent] = deduped; - } else { - result[item.intent] = ''; - } - - triggerRules[fileId] = { ...triggerRules[fileId], ...result }; - }); - } - }); - - const crossTrainConfig: ICrossTrainConfig = { - botName: botName, - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - }; - crossTrainConfig.rootIds = keys(countMap).filter( - (key) => (countMap[key] === 0 || key === rootId) && triggerRules[key] - ); - crossTrainConfig.triggerRules = triggerRules; - return addLocaleToConfig(crossTrainConfig, luFiles); -} diff --git a/extensions/azurePublish/src/utils/fileUtil.ts b/extensions/azurePublish/src/utils/fileUtil.ts deleted file mode 100644 index 0e5ab3af4a..0000000000 --- a/extensions/azurePublish/src/utils/fileUtil.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -export function getExtension(filename?: string): string | any { - if (typeof filename !== 'string') return filename; - return filename.substring(filename.lastIndexOf('.') + 1, filename.length) || filename; -} - -export function getBaseName(filename: string, sep?: string): string | any { - if (sep) return filename.substr(0, filename.lastIndexOf(sep)); - return filename.substring(0, filename.lastIndexOf('.')) || filename; -} diff --git a/extensions/azurePublish/src/utils/jsonWalk.ts b/extensions/azurePublish/src/utils/jsonWalk.ts deleted file mode 100644 index 6f4efcd4e1..0000000000 --- a/extensions/azurePublish/src/utils/jsonWalk.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * visitor function used by JsonWalk - * @param path jsonPath string - * @param value current node value - * @return boolean, true to stop walk deep - */ -export interface VisitorFunc { - (path: string, value: any): boolean; -} - -/** - * - * @param path jsonPath string - * @param value current node value - * @param visitor - */ - -export const JsonWalk = (path: string, value: any, visitor: VisitorFunc) => { - const stop = visitor(path, value); - if (stop === true) return; - - // extract array - if (Array.isArray(value)) { - value.forEach((child, index) => { - JsonWalk(`${path}[${index}]`, child, visitor); - }); - - // extract object - } else if (typeof value === 'object' && value) { - Object.keys(value).forEach((key) => { - JsonWalk(`${path}.${key}`, value[key], visitor); - }); - } -}; From 4d92bf23fc9051ca259018d79b4f6cbb768da23b Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 30 Nov 2020 15:40:16 +0800 Subject: [PATCH 2/8] update the orchestrator path --- Composer/packages/server/src/models/bot/builder.ts | 11 +++++++++-- extensions/azurePublish/src/index.ts | 1 - 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index 61f1c8df9a..1a53c2b87e 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -8,6 +8,7 @@ import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathemati import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler'; import { luImportResolverGenerator, getLUFiles, getQnAFiles } from '@bfc/shared/lib/luBuildResolver'; import { Orchestrator } from '@microsoft/bf-orchestrator'; +import keys from 'lodash/keys'; import { Path } from '../../utility/path'; import { IFileStorage } from '../storage/interface'; @@ -26,6 +27,7 @@ const SETTINGS = 'settings'; const INTERRUPTION = 'interruption'; const SAMPLE_SIZE_CONFIGURATION = 2; const CrossTrainConfigName = 'cross-train.config.json'; +const MODLE = 'model'; export type SingleConfig = { rootDialog: boolean; @@ -236,16 +238,21 @@ export class Builder { const defaultNLR = nlrList.default; const folderName = defaultNLR.replace('.onnx', ''); const modelPath = Path.resolve(await this.getModelPathAsync(), folderName); - const destDir = Path.resolve(this.generatedFolderPath, folderName); + const destDir = Path.resolve(Path.join(this.botDir, MODLE), folderName); await copy(modelPath, destDir); await this.updateOrchestratorSetting(folderName); } } private async updateOrchestratorSetting(dirName: string) { + const runtimeRootPath = './ComposerDialogs'; const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json'); const content = JSON.parse(await this.storage.readFile(settingPath)); - content.orchestrator.ModelPath = './ComposerDialogs/generated/' + dirName; + content.orchestrator.ModelPath = `${runtimeRootPath}/${MODLE}/${dirName}`; + keys(content.orchestrator.snapshots).forEach((key) => { + const values = content.orchestrator.snapshots[key].split('ComposerDialogs'); + content.orchestrator.snapshots[key] = `${runtimeRootPath}${values[1]}`; + }); await this.storage.writeFile(settingPath, JSON.stringify(content, null, 2)); } diff --git a/extensions/azurePublish/src/index.ts b/extensions/azurePublish/src/index.ts index 36f2059718..4b22f0ec9a 100644 --- a/extensions/azurePublish/src/index.ts +++ b/extensions/azurePublish/src/index.ts @@ -127,7 +127,6 @@ export default async (composer: IExtensionRegistration): Promise => { private init = async (project: any, srcTemplate: string, resourcekey: string, runtime: any) => { // point to the declarative assets (possibly in remote storage) const botFiles = project.getProject().files; - console.log(botFiles); const botFolder = this.getBotFolder(resourcekey, this.mode); const runtimeFolder = this.getRuntimeFolder(resourcekey); From 5748848ec43b1e5ad3869659e256e1c337d2de92 Mon Sep 17 00:00:00 2001 From: leilzh Date: Mon, 30 Nov 2020 15:42:11 +0800 Subject: [PATCH 3/8] update the cleanup --- extensions/azurePublish/src/index.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/extensions/azurePublish/src/index.ts b/extensions/azurePublish/src/index.ts index 4b22f0ec9a..3887859ab1 100644 --- a/extensions/azurePublish/src/index.ts +++ b/extensions/azurePublish/src/index.ts @@ -165,13 +165,13 @@ export default async (composer: IExtensionRegistration): Promise => { * @param resourcekey */ private async cleanup(resourcekey: string) { - // try { - // const projFolder = this.getRuntimeFolder(resourcekey); - // await emptyDir(projFolder); - // await rmdir(projFolder); - // } catch (error) { - // this.logger('$O', error); - // } + try { + const projFolder = this.getRuntimeFolder(resourcekey); + await emptyDir(projFolder); + await rmdir(projFolder); + } catch (error) { + this.logger('$O', error); + } } /** From b87c506a104569ea8b835dd58e5d09c5f820ad6c Mon Sep 17 00:00:00 2001 From: leilzh Date: Tue, 1 Dec 2020 14:07:05 +0800 Subject: [PATCH 4/8] fix typo --- Composer/packages/server/src/models/bot/builder.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index 1a53c2b87e..3ed0a3fe41 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -27,7 +27,7 @@ const SETTINGS = 'settings'; const INTERRUPTION = 'interruption'; const SAMPLE_SIZE_CONFIGURATION = 2; const CrossTrainConfigName = 'cross-train.config.json'; -const MODLE = 'model'; +const MODEL = 'model'; export type SingleConfig = { rootDialog: boolean; @@ -238,7 +238,7 @@ export class Builder { const defaultNLR = nlrList.default; const folderName = defaultNLR.replace('.onnx', ''); const modelPath = Path.resolve(await this.getModelPathAsync(), folderName); - const destDir = Path.resolve(Path.join(this.botDir, MODLE), folderName); + const destDir = Path.resolve(Path.join(this.botDir, MODEL), folderName); await copy(modelPath, destDir); await this.updateOrchestratorSetting(folderName); } @@ -248,7 +248,7 @@ export class Builder { const runtimeRootPath = './ComposerDialogs'; const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json'); const content = JSON.parse(await this.storage.readFile(settingPath)); - content.orchestrator.ModelPath = `${runtimeRootPath}/${MODLE}/${dirName}`; + content.orchestrator.ModelPath = `${runtimeRootPath}/${MODEL}/${dirName}`; keys(content.orchestrator.snapshots).forEach((key) => { const values = content.orchestrator.snapshots[key].split('ComposerDialogs'); content.orchestrator.snapshots[key] = `${runtimeRootPath}${values[1]}`; From 116b8e46c5de73fd0004efc79566de2b578e3e74 Mon Sep 17 00:00:00 2001 From: leilzh Date: Tue, 1 Dec 2020 18:56:38 +0800 Subject: [PATCH 5/8] catch the build error --- extensions/azurePublish/src/deploy.ts | 11 ++++++++++- extensions/azurePublish/src/luisAndQnA.ts | 2 -- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/extensions/azurePublish/src/deploy.ts b/extensions/azurePublish/src/deploy.ts index a5224c2c1f..b0a354f4a2 100644 --- a/extensions/azurePublish/src/deploy.ts +++ b/extensions/azurePublish/src/deploy.ts @@ -59,7 +59,16 @@ export class BotProjectDeploy { language = 'en-us'; } - build(project, this.projPath, settings); + this.logger({ + status: BotProjectDeployLoggerType.DEPLOY_INFO, + message: "Building the bot's resources ...", + }); + await build(project, this.projPath, settings); + + this.logger({ + status: BotProjectDeployLoggerType.DEPLOY_INFO, + message: 'Build Success!', + }); // this function returns an object that contains the luis APP ids mapping // each dialog to its matching app. diff --git a/extensions/azurePublish/src/luisAndQnA.ts b/extensions/azurePublish/src/luisAndQnA.ts index 140ee200da..d5bc123433 100644 --- a/extensions/azurePublish/src/luisAndQnA.ts +++ b/extensions/azurePublish/src/luisAndQnA.ts @@ -101,10 +101,8 @@ export async function publishLuisToPrediction( const response = await rp.get(getAccountUri, options); // this should include an array of account info objects - console.log(response); accountList = JSON.parse(response); } catch (err) { - console.log(err); // handle the token invalid const error = JSON.parse(err.error); if (error?.error?.message && error?.error?.message.indexOf('access token expiry') > 0) { From a685725b3191e0ae4fcb061dd8deb2be183e9b39 Mon Sep 17 00:00:00 2001 From: leilzh Date: Tue, 1 Dec 2020 19:31:05 +0800 Subject: [PATCH 6/8] remove qna endpoint --- Composer/packages/server/src/models/bot/builder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index 3ed0a3fe41..4402be42bf 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -94,6 +94,7 @@ export class Builder { await this.runQnaBuild(interruptionQnaFiles); await this.runOrchestratorBuild(orchestratorBuildFiles); } catch (error) { + console.log(error); throw new Error(error.message ?? error.text ?? 'Error publishing to LUIS or QNA.'); } }; @@ -407,8 +408,7 @@ export class Builder { }); if (qnaContents) { - const subscriptionKeyEndpoint = - config.endpoint ?? `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; + const subscriptionKeyEndpoint = `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; const buildResult = await this.qnaBuilder.build(qnaContents, config.subscriptionKey, config.botName, { endpoint: subscriptionKeyEndpoint, From 9f1ba22c0f0a225bb38e10b8083fe06299948f10 Mon Sep 17 00:00:00 2001 From: leilzh Date: Tue, 1 Dec 2020 19:41:03 +0800 Subject: [PATCH 7/8] remove console --- Composer/packages/server/src/models/bot/builder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index 4402be42bf..d0ec7d8826 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -94,7 +94,6 @@ export class Builder { await this.runQnaBuild(interruptionQnaFiles); await this.runOrchestratorBuild(orchestratorBuildFiles); } catch (error) { - console.log(error); throw new Error(error.message ?? error.text ?? 'Error publishing to LUIS or QNA.'); } }; From c8abbc634f8d92b220febee462c72c77a8512cd4 Mon Sep 17 00:00:00 2001 From: leilzh Date: Wed, 2 Dec 2020 17:37:33 +0800 Subject: [PATCH 8/8] add qnaconfig --- .../packages/server/src/models/bot/builder.ts | 19 +++++++++++++++++++ extensions/azurePublish/src/deploy.ts | 4 +++- extensions/azurePublish/src/luisAndQnA.ts | 1 - 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index d0ec7d8826..f8b031cab8 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -244,6 +244,25 @@ export class Builder { } } + public async getQnaConfig() { + const config = this._getConfig([]); + const subscriptionKeyEndpoint = `https://${config?.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; + // Find any files that contain the name 'qnamaker.settings' in them + // These are generated by the LuBuild process and placed in the generated folder + // These contain dialog-to-luis app id mapping + const paths = await this.storage.glob('qnamaker.settings.*', this.generatedFolderPath); + if (!paths.length) return {}; + + const qnaConfigFile = await this.storage.readFile(Path.join(this.generatedFolderPath, paths[0])); + const qna: any = {}; + + const qnaConfig = await JSON.parse(qnaConfigFile); + const endpointKey = await this.qnaBuilder.getEndpointKeys(config.subscriptionKey, subscriptionKeyEndpoint); + Object.assign(qna, qnaConfig.qna, { endpointKey: endpointKey.primaryEndpointKey }); + + return qna; + } + private async updateOrchestratorSetting(dirName: string) { const runtimeRootPath = './ComposerDialogs'; const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json'); diff --git a/extensions/azurePublish/src/deploy.ts b/extensions/azurePublish/src/deploy.ts index b0a354f4a2..028a3c237c 100644 --- a/extensions/azurePublish/src/deploy.ts +++ b/extensions/azurePublish/src/deploy.ts @@ -72,7 +72,7 @@ export class BotProjectDeploy { // this function returns an object that contains the luis APP ids mapping // each dialog to its matching app. - const { luisAppIds, qnaConfig } = await publishLuisToPrediction( + const luisAppIds = await publishLuisToPrediction( name, environment, this.accessToken, @@ -82,6 +82,8 @@ export class BotProjectDeploy { this.logger ); + const qnaConfig = await project.builder.getQnaConfig(); + // amend luis settings with newly generated values settings.luis = { ...settings.luis, diff --git a/extensions/azurePublish/src/luisAndQnA.ts b/extensions/azurePublish/src/luisAndQnA.ts index d5bc123433..aa6f788661 100644 --- a/extensions/azurePublish/src/luisAndQnA.ts +++ b/extensions/azurePublish/src/luisAndQnA.ts @@ -94,7 +94,6 @@ export async function publishLuisToPrediction( // DOCS HERE: https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5be313cec181ae720aa2b26c // This returns a list of azure account information objects with AzureSubscriptionID, ResourceGroup, AccountName for each. const getAccountUri = `${luisEndpoint}/luis/api/v2.0/azureaccounts`; - console.log(getAccountUri); const options = { headers: { Authorization: `Bearer ${accessToken}`, 'Ocp-Apim-Subscription-Key': luisAuthoringKey }, } as rp.RequestPromiseOptions;