feat: Add cross train before luis publish

Composer/packages/client/package.json

@@ -20,7 +20,7 @@
     "@bfc/extensions": "*",
     "@bfc/indexers": "*",
     "@bfc/shared": "*",
-    "@bfcomposer/bf-lu": "^1.2.5",
+    "@bfcomposer/bf-lu": "^1.2.9",
     "@emotion/core": "^10.0.7",
     "@reach/router": "^1.2.1",
     "axios": "^0.18.0",

Composer/packages/extensions/obiformeditor/src/Form/fields/PromptField/UserInput.tsx

@@ -8,6 +8,7 @@ import { FieldProps } from '@bfcomposer/react-jsonschema-form';
 import formatMessage from 'format-message';
 import { JSONSchema6 } from 'json-schema';
 import { SDKTypes, MicrosoftInputDialog, ChoiceInput, ConfirmInput } from '@bfc/shared';
+import { DialogInfo } from '@bfc/indexers/lib/type';
 
 import { TextWidget, SelectWidget } from '../../widgets';
 import { LuEditorWidget } from '../../widgets/LuEditorWidget';
@@ -16,7 +17,6 @@ import { field } from './styles';
 import { GetSchema, PromptFieldChangeHandler } from './types';
 import { ChoiceInputSettings } from './ChoiceInput';
 import { ConfirmInputSettings } from './ConfirmInput';
-import { DialogInfo } from '@bfc/indexers/lib/type';
 
 const getOptions = (enumSchema: JSONSchema6) => {
   if (!enumSchema || !enumSchema.enum || !Array.isArray(enumSchema.enum)) {

Composer/packages/extensions/obiformeditor/src/Form/widgets/LuEditorWidget.tsx

@@ -9,6 +9,7 @@ import { LuFile, filterSectionDiagnostics } from '@bfc/indexers';
 import { LuIntentSection } from '@bfc/shared';
 
 import { FormContext } from '../types';
+
 import { WidgetLabel } from './WidgetLabel';
 
 interface LuEditorWidgetProps {

Composer/packages/lib/indexers/package.json

@@ -29,7 +29,7 @@
   },
   "dependencies": {
     "@bfc/shared": "*",
-    "@bfcomposer/bf-lu": "^1.2.5",
+    "@bfcomposer/bf-lu": "^1.2.9",
     "botbuilder-lg": "^4.8.0-preview.106823",
     "botframework-expressions": "^4.8.0-preview.106476",
     "lodash": "^4.17.15"

Composer/packages/lib/indexers/src/dialogIndexer.ts

@@ -12,7 +12,7 @@ import { JsonWalk, VisitorFunc } from './utils/jsonWalk';
 import { getBaseName } from './utils/help';
 import { Diagnostic } from './diagnostic';
 import ExtractMemoryPaths from './dialogUtils/extractMemoryPaths';
-
+import ExtractIntentTriggers from './dialogUtils/extractIntentTriggers';
 // find out all lg templates given dialog
 function ExtractLgTemplates(id, dialog): LgTemplateJsonPath[] {
   const templates: LgTemplateJsonPath[] = [];
@@ -95,7 +95,7 @@ function ExtractLuIntents(dialog, id: string): ReferredLuIntents[] {
 
 // find out all triggers given dialog
 function ExtractTriggers(dialog): ITrigger[] {
-  const trigers: ITrigger[] = [];
+  const triggers: ITrigger[] = [];
   /**    *
    * @param path , jsonPath string
    * @param value , current node value    *
@@ -118,15 +118,15 @@ function ExtractTriggers(dialog): ITrigger[] {
           } else if (trigger.isIntent && has(rule, 'intent')) {
             trigger.displayName = rule.intent;
           }
-          trigers.push(trigger);
+          triggers.push(trigger);
         }
       });
       return true;
     }
     return false;
   };
   JsonWalk('$', dialog, visitor);
-  return trigers;
+  return triggers;
 }
 
 // find out all referred dialog
@@ -204,6 +204,7 @@ function parse(id: string, content: any, schema: any) {
     luFile: getBaseName(luFile, '.lu'),
     lgFile: getBaseName(lgFile, '.lg'),
     triggers: ExtractTriggers(content),
+    intentTriggers: ExtractIntentTriggers(content),
   };
 }
 

Composer/packages/lib/indexers/src/dialogUtils/extractIntentTriggers.ts

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { SDKTypes } from '@bfc/shared';
+
+import { VisitorFunc, JsonWalk } from '../utils/jsonWalk';
+
+import { IIntentTrigger } from './types';
+
+// find out all properties from given dialog
+function ExtractIntentTriggers(value: any): IIntentTrigger[] {
+  const triggers: IIntentTrigger[] = [];
+
+  const visitor: VisitorFunc = (path: string, value: any): boolean => {
+    if (value?.$type === SDKTypes.OnIntent) {
+      if (value.intent) {
+        const dialogs: string[] = [];
+
+        const visitor: VisitorFunc = (path: string, value: any): boolean => {
+          if (value?.$type === SDKTypes.BeginDialog) {
+            if (value.dialog) {
+              dialogs.push(value.dialog);
+            }
+            return true;
+          }
+          return false;
+        };
+        JsonWalk('$', value, visitor);
+        if (dialogs.length) {
+          triggers.push({
+            intent: value.intent,
+            dialogs,
+          });
+        }
+      }
+      return true;
+    }
+    return false;
+  };
+  JsonWalk('$', value, visitor);
+
+  return triggers;
+}
+
+export default ExtractIntentTriggers;

Composer/packages/lib/indexers/src/dialogUtils/types.ts

@@ -23,4 +23,9 @@ export interface IExpressionProperties {
   };
 }
 
+export interface IIntentTrigger {
+  intent: string;
+  dialogs: string[];
+}
+
 export type CheckerFunc = (path: string, value: any, type: string, schema: any) => Diagnostic[] | null; // error msg

Composer/packages/lib/indexers/src/luIndexer.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser';
+import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser/composerindex';
 import get from 'lodash/get';
 import { LuIntentSection } from '@bfc/shared';
 

Composer/packages/lib/indexers/src/type.ts

@@ -4,6 +4,7 @@
 import { LuIntentSection } from '@bfc/shared';
 
 import { Diagnostic } from './diagnostic';
+import { IIntentTrigger } from './dialogUtils/types';
 
 export interface FileInfo {
   name: string;
@@ -38,6 +39,7 @@ export interface DialogInfo {
   relativePath: string;
   userDefinedVariables: string[];
   triggers: ITrigger[];
+  intentTriggers: IIntentTrigger[];
 }
 
 export interface LgTemplateJsonPath {

Composer/packages/lib/indexers/src/types/bf-lu.d.ts

@@ -1,7 +1,7 @@
 /// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-declare module '@bfcomposer/bf-lu/lib/parser' {
+declare module '@bfcomposer/bf-lu/lib/parser/composerindex' {
   namespace parser {
     function parseFile(fileContent: any, log: any, locale: any): any;
     function validateLUISBlob(LUISJSONBlob: any): any;

Composer/packages/lib/indexers/src/utils/luUtil.ts

@@ -7,7 +7,7 @@
  * for more usage detail, please check client/__tests__/utils/luUtil.test.ts
  */
 
-import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser';
+import { sectionHandler } from '@bfcomposer/bf-lu/lib/parser/composerindex';
 import isEmpty from 'lodash/isEmpty';
 import { LuIntentSection } from '@bfc/shared';
 

Composer/packages/server/package.json

@@ -57,7 +57,7 @@
     "@bfc/lg-languageserver": "*",
     "@bfc/lu-languageserver": "*",
     "@bfc/shared": "*",
-    "@bfcomposer/bf-lu": "^1.2.5",
+    "@bfcomposer/bf-lu": "^1.2.9",
     "archiver": "^3.0.0",
     "axios": "^0.18.0",
     "azure-storage": "^2.10.3",

Composer/packages/server/src/models/bot/botProject.ts

@@ -23,7 +23,7 @@ import { ISettingManager, OBFUSCATED_VALUE } from '../settings';
 import log from '../../logger';
 
 import { IFileStorage } from './../storage/interface';
-import { LocationRef, LuisStatus, FileUpdateType } from './interface';
+import { LocationRef } from './interface';
 import { LuPublisher } from './luPublisher';
 import { DialogSetting } from './interface';
 
@@ -89,7 +89,6 @@ export class BotProject {
     if (this.settings) {
       await this.luPublisher.setLuisConfig(this.settings.luis);
     }
-    await this.luPublisher.loadStatus(this.luFiles.map(f => f.relativePath));
   };
 
   public getIndexes = () => {
@@ -99,7 +98,7 @@ export class BotProject {
       location: this.dir,
       dialogs: this.dialogs,
       lgFiles: this.lgFiles,
-      luFiles: this.mergeLuStatus(this.luFiles, this.luPublisher.status),
+      luFiles: this.luFiles,
       schemas: this.getSchemas(),
       botEnvironment: this.environment.getEnvironmentName(this.name),
       settings: this.settings,
@@ -133,25 +132,6 @@ export class BotProject {
     await this.luPublisher.setLuisConfig(config.luis);
   };
 
-  // merge the status managed by luPublisher to the LuFile structure to keep a
-  // unified interface
-  private mergeLuStatus = (
-    luFiles: LuFile[],
-    luStatus: {
-      [key: string]: LuisStatus;
-    }
-  ) => {
-    return luFiles.map(x => {
-      if (!luStatus[x.relativePath]) {
-        throw new Error(`No luis status for lu file ${x.relativePath}`);
-      }
-      return {
-        ...x,
-        status: luStatus[x.relativePath],
-      };
-    });
-  };
-
   public getSchemas = () => {
     let editorSchema = this.defaultEditorSchema;
     let sdkSchema = this.defaultSDKSchema;
@@ -305,9 +285,8 @@ export class BotProject {
     }
 
     await this._updateFile(luFile.relativePath, content);
-    await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.UPDATE);
 
-    return this.mergeLuStatus(this.luFiles, this.luPublisher.status);
+    return this.luFiles;
   };
 
   public createLuFile = async (id: string, content: string, dir: string = this.defaultDir(id)): Promise<LuFile[]> => {
@@ -319,8 +298,7 @@ export class BotProject {
 
     // TODO: validate before save
     await this._createFile(relativePath, content);
-    await this.luPublisher.onFileChange(relativePath, FileUpdateType.CREATE); // let publisher know that some files changed
-    return this.mergeLuStatus(this.luFiles, this.luPublisher.status); // return a merged LUFile always
+    return this.luFiles; // return a merged LUFile always
   };
 
   public removeLuFile = async (id: string): Promise<LuFile[]> => {
@@ -331,41 +309,31 @@ export class BotProject {
 
     await this._removeFile(luFile.relativePath);
 
-    await this.luPublisher.onFileChange(luFile.relativePath, FileUpdateType.DELETE);
     await this._cleanUp(luFile.relativePath);
-    return this.mergeLuStatus(this.luFiles, this.luPublisher.status);
+    return this.luFiles;
   };
 
   public publishLuis = async (authoringKey: string) => {
     this.luPublisher.setAuthoringKey(authoringKey);
     const referred = this.luFiles.filter(this.isReferred);
-    const unpublished = this.luPublisher.getUnpublisedFiles(referred);
 
-    const invalidLuFile = unpublished.filter(file => file.diagnostics.length !== 0);
+    const invalidLuFile = referred.filter(file => file.diagnostics.length !== 0);
     if (invalidLuFile.length !== 0) {
       const msg = this.generateErrorMessage(invalidLuFile);
       throw new Error(`The Following LuFile(s) are invalid: \n` + msg);
     }
-    const emptyLuFiles = unpublished.filter(this.isLuFileEmpty);
+    const emptyLuFiles = referred.filter(this.isLuFileEmpty);
     if (emptyLuFiles.length !== 0) {
       const msg = emptyLuFiles.map(file => file.id).join(' ');
       throw new Error(`You have the following empty LuFile(s): ` + msg);
     }
 
-    if (unpublished.length > 0) {
-      await this.luPublisher.publish(unpublished);
+    if (referred.length > 0) {
+      this.luPublisher.createCrossTrainConfig(this.dialogs, referred);
+      await this.luPublisher.publish(referred);
     }
 
-    return this.mergeLuStatus(this.luFiles, this.luPublisher.status);
-  };
-
-  public checkLuisPublished = () => {
-    const referredLuFiles = this.luFiles.filter(this.isReferred);
-    if (referredLuFiles.length <= 0) {
-      return true;
-    } else {
-      return this.luPublisher.checkLuisPublised(referredLuFiles);
-    }
+    return this.luFiles;
   };
 
   public cloneFiles = async (locationRef: LocationRef): Promise<LocationRef> => {
@@ -522,7 +490,7 @@ export class BotProject {
       // load only from the data dir, otherwise may get "build" versions from
       // deployment process
       const root = this.dataDir;
-      const paths = await this.fileStorage.glob(pattern, root);
+      const paths = await this.fileStorage.glob([pattern, '!(generated/**)'], root);
 
       for (const filePath of paths.sort()) {
         const realFilePath: string = Path.join(root, filePath);