feat: azure publish with orchestrator

Composer/packages/client/src/recoilModel/dispatchers/publisher.ts

@@ -13,8 +13,12 @@ import {
   botLoadErrorState,
   isEjectRuntimeExistState,
   filePersistenceState,
+  luFilesState,
+  qnaFilesState,
 } from '../atoms/botState';
 import { botEndpointsState } from '../atoms';
+import { dialogsSelectorFamily } from '../selectors';
+import * as luUtil from '../../utils/luUtil';
 
 import { BotStatus, Text } from './../../constants';
 import httpClient from './../../utils/httpUtil';
@@ -121,10 +125,24 @@ export const publisherDispatcher = () => {
   });
 
   const publishToTarget = useRecoilCallback(
-    (callbackHelpers: CallbackInterface) => async (projectId: string, target: any, metadata, sensitiveSettings) => {
+    (callbackHelpers: CallbackInterface) => async (
+      projectId: string,
+      target: any,
+      metadata: any,
+      sensitiveSettings
+    ) => {
       try {
+        const { snapshot } = callbackHelpers;
+        const dialogs = await snapshot.getPromise(dialogsSelectorFamily(projectId));
+        const luFiles = await snapshot.getPromise(luFilesState(projectId));
+        const qnaFiles = await snapshot.getPromise(qnaFilesState(projectId));
+        const referredLuFiles = luUtil.checkLuisBuild(luFiles, dialogs);
         const response = await httpClient.post(`/publish/${projectId}/publish/${target.name}`, {
-          metadata,
+          metadata: {
+            ...metadata,
+            luResources: referredLuFiles.map((file) => file.id),
+            qnaResources: qnaFiles.map((file) => file.id),
+          },
           sensitiveSettings,
         });
         await publishSuccess(callbackHelpers, projectId, response.data, target);

Composer/packages/server/src/models/bot/botProject.ts

@@ -494,6 +494,7 @@ export class BotProject implements IBotProject {
         }
       });
 
+      this.builder.rootDir = this.dir;
       this.builder.setBuildConfig(
         { ...luisConfig, subscriptionKey: qnaConfig.subscriptionKey, qnaRegion: qnaConfig.qnaRegion },
         this.settings.downsampling

Composer/packages/server/src/models/bot/builder.ts

@@ -2,12 +2,13 @@
 // Licensed under the MIT License.
 
 /* eslint-disable @typescript-eslint/no-var-requires */
-import { pathExists, writeFile } from 'fs-extra';
+import { pathExists, writeFile, copy } from 'fs-extra';
 import { FileInfo, IConfig, SDKKinds } from '@bfc/shared';
 import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler';
 import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler';
 import { luImportResolverGenerator, getLUFiles, getQnAFiles } from '@bfc/shared/lib/luBuildResolver';
 import { Orchestrator } from '@microsoft/bf-orchestrator';
+import keys from 'lodash/keys';
 
 import { Path } from '../../utility/path';
 import { IFileStorage } from '../storage/interface';
@@ -26,6 +27,7 @@ const SETTINGS = 'settings';
 const INTERRUPTION = 'interruption';
 const SAMPLE_SIZE_CONFIGURATION = 2;
 const CrossTrainConfigName = 'cross-train.config.json';
+const MODEL = 'model';
 
 export type SingleConfig = {
   rootDialog: boolean;
@@ -51,6 +53,7 @@ export class Builder {
   public config: IConfig | null = null;
   public downSamplingConfig: DownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 };
   private _locale: string;
+  private containOrchestrator = false;
 
   private luBuilder = new luBuild.Builder((message) => {
     log(message);
@@ -67,6 +70,12 @@ export class Builder {
     this._locale = locale;
   }
 
+  public set rootDir(path: string) {
+    this.botDir = path;
+    this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER);
+    this.interruptionFolderPath = Path.join(this.generatedFolderPath, INTERRUPTION);
+  }
+
   public build = async (luFiles: FileInfo[], qnaFiles: FileInfo[], allFiles: FileInfo[]) => {
     try {
       await this.createGeneratedDir();
@@ -76,7 +85,11 @@ export class Builder {
 
       const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles();
       const { luBuildFiles, orchestratorBuildFiles } = this.separateLuFiles(interruptionLuFiles, allFiles);
-
+      if (orchestratorBuildFiles.length) {
+        this.containOrchestrator = true;
+      } else {
+        this.containOrchestrator = false;
+      }
       await this.runLuBuild(luBuildFiles);
       await this.runQnaBuild(interruptionQnaFiles);
       await this.runOrchestratorBuild(orchestratorBuildFiles);
@@ -219,6 +232,49 @@ export class Builder {
     return await Orchestrator.buildAsync(modelPath, luObjects, isDialog, null, fullEmbedding);
   }
 
+  public async copyModelPathToBot() {
+    if (this.containOrchestrator) {
+      const nlrList = await this.runOrchestratorNlrList();
+      const defaultNLR = nlrList.default;
+      const folderName = defaultNLR.replace('.onnx', '');
+      const modelPath = Path.resolve(await this.getModelPathAsync(), folderName);
+      const destDir = Path.resolve(Path.join(this.botDir, MODEL), folderName);
+      await copy(modelPath, destDir);
+      await this.updateOrchestratorSetting(folderName);
+    }
+  }
+
+  public async getQnaConfig() {
+    const config = this._getConfig([]);
+    const subscriptionKeyEndpoint = `https://${config?.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`;
+    // Find any files that contain the name 'qnamaker.settings' in them
+    // These are generated by the LuBuild process and placed in the generated folder
+    // These contain dialog-to-luis app id mapping
+    const paths = await this.storage.glob('qnamaker.settings.*', this.generatedFolderPath);
+    if (!paths.length) return {};
+
+    const qnaConfigFile = await this.storage.readFile(Path.join(this.generatedFolderPath, paths[0]));
+    const qna: any = {};
+
+    const qnaConfig = await JSON.parse(qnaConfigFile);
+    const endpointKey = await this.qnaBuilder.getEndpointKeys(config.subscriptionKey, subscriptionKeyEndpoint);
+    Object.assign(qna, qnaConfig.qna, { endpointKey: endpointKey.primaryEndpointKey });
+
+    return qna;
+  }
+
+  private async updateOrchestratorSetting(dirName: string) {
+    const runtimeRootPath = './ComposerDialogs';
+    const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json');
+    const content = JSON.parse(await this.storage.readFile(settingPath));
+    content.orchestrator.ModelPath = `${runtimeRootPath}/${MODEL}/${dirName}`;
+    keys(content.orchestrator.snapshots).forEach((key) => {
+      const values = content.orchestrator.snapshots[key].split('ComposerDialogs');
+      content.orchestrator.snapshots[key] = `${runtimeRootPath}${values[1]}`;
+    });
+    await this.storage.writeFile(settingPath, JSON.stringify(content, null, 2));
+  }
+
   private async createGeneratedDir() {
     // clear previous folder
     await this.deleteDir(this.generatedFolderPath);
@@ -370,8 +426,7 @@ export class Builder {
     });
 
     if (qnaContents) {
-      const subscriptionKeyEndpoint =
-        config.endpoint ?? `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`;
+      const subscriptionKeyEndpoint = `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`;
 
       const buildResult = await this.qnaBuilder.build(qnaContents, config.subscriptionKey, config.botName, {
         endpoint: subscriptionKeyEndpoint,

extensions/azurePublish/package.json

@@ -20,8 +20,6 @@
     "@azure/ms-rest-nodeauth": "3.0.3",
     "@bfc/indexers": "../../Composer/packages/lib/indexers",
     "@bfc/shared": "../../Composer/packages/lib/shared",
-    "@microsoft/bf-lu": "4.11.0-dev.20201005.7e5e1b8",
-    "@microsoft/bf-luis-cli": "^4.10.0-dev.20200721.8bb21ac",
     "adal-node": "0.2.1",
     "archiver": "^5.0.2",
     "fs-extra": "8.1.0",

extensions/azurePublish/src/deploy.ts

@@ -8,7 +8,7 @@ import * as rp from 'request-promise';
 
 import { BotProjectDeployConfig } from './botProjectDeployConfig';
 import { BotProjectDeployLoggerType } from './botProjectLoggerType';
-import { LuisAndQnaPublish } from './luisAndQnA';
+import { build, publishLuisToPrediction } from './luisAndQnA';
 import archiver = require('archiver');
 
 export class BotProjectDeploy {
@@ -58,20 +58,32 @@ export class BotProjectDeploy {
       if (!language) {
         language = 'en-us';
       }
-      const publisher = new LuisAndQnaPublish({ logger: this.logger, projPath: this.projPath });
+
+      this.logger({
+        status: BotProjectDeployLoggerType.DEPLOY_INFO,
+        message: "Building the bot's resources ...",
+      });
+      await build(project, this.projPath, settings);
+
+      this.logger({
+        status: BotProjectDeployLoggerType.DEPLOY_INFO,
+        message: 'Build Success!',
+      });
 
       // this function returns an object that contains the luis APP ids mapping
       // each dialog to its matching app.
-      const { luisAppIds, qnaConfig } = await publisher.publishLuisAndQna(
+      const luisAppIds = await publishLuisToPrediction(
         name,
         environment,
         this.accessToken,
-        language,
         settings.luis,
-        settings.qna,
-        luisResource
+        luisResource,
+        this.projPath,
+        this.logger
       );
 
+      const qnaConfig = await project.builder.getQnaConfig();
+
       // amend luis settings with newly generated values
       settings.luis = {
         ...settings.luis,

extensions/azurePublish/src/index.ts

@@ -126,7 +126,7 @@ export default async (composer: IExtensionRegistration): Promise<void> => {
      */
     private init = async (project: any, srcTemplate: string, resourcekey: string, runtime: any) => {
       // point to the declarative assets (possibly in remote storage)
-      const botFiles = project.getProject().filesWithoutRecognizers;
+      const botFiles = project.getProject().files;
       const botFolder = this.getBotFolder(resourcekey, this.mode);
       const runtimeFolder = this.getRuntimeFolder(resourcekey);
 
@@ -321,6 +321,8 @@ export default async (composer: IExtensionRegistration): Promise<void> => {
         defaultLanguage,
         settings,
         accessToken,
+        luResources,
+        qnaResources
       } = config;
 
       // get the appropriate runtime template which contains methods to build and configure the runtime
@@ -347,7 +349,7 @@ export default async (composer: IExtensionRegistration): Promise<void> => {
         // Merge all the settings
         // this combines the bot-wide settings, the environment specific settings, and 2 new fields needed for deployed bots
         // these will be written to the appropriate settings file inside the appropriate runtime plugin.
-        const mergedSettings = mergeDeep(fullSettings, settings);
+        const mergedSettings = mergeDeep(fullSettings, settings, {luResources, qnaResources});
 
         // Prepare parameters and then perform the actual deployment action
         const customizeConfiguration: CreateAndDeployResources = {
@@ -392,6 +394,7 @@ export default async (composer: IExtensionRegistration): Promise<void> => {
      * plugin methods
      *************************************************************************************************/
     publish = async (config: PublishConfig, project: IBotProject, metadata, user) => {
+      const {luResources, qnaResources} = metadata;
       const {
         // these are provided by Composer
         profileName, // the name of the publishing profile "My Azure Prod Slot"
@@ -436,7 +439,7 @@ export default async (composer: IExtensionRegistration): Promise<void> => {
           throw new Error('Required field `settings` is missing from publishing profile.');
         }
 
-        this.asyncPublish(config, project, resourcekey, jobId);
+        this.asyncPublish({...config, luResources, qnaResources}, project, resourcekey, jobId);
       } catch (err) {
         this.logger('%O', err);
         if (err instanceof Error) {