feat: suggest user defined memory variables in LG files

Composer/packages/tools/language-servers/language-generation/package.json

@@ -18,6 +18,7 @@
     "@bfc/built-in-functions": "*",
     "@bfc/indexers": "*",
     "botbuilder-lg": "4.12.0-rc1",
+    "adaptive-expressions": "4.12.0-rc1",
     "vscode-languageserver": "^5.3.0-next"
   },
   "devDependencies": {

Composer/packages/tools/language-servers/language-generation/src/LGServer.ts

@@ -21,6 +21,8 @@ import {
   FoldingRange,
 } from 'vscode-languageserver-protocol';
 import get from 'lodash/get';
+import uniq from 'lodash/uniq';
+import merge from 'lodash/merge';
 import isEqual from 'lodash/isEqual';
 import { filterTemplateDiagnostics, isValid, lgUtil } from '@bfc/indexers';
 import { MemoryResolver, ResolverResource, LgFile } from '@bfc/shared';
@@ -55,14 +57,20 @@ export class LGServer {
   private _lgParser = new LgParser();
   private _luisEntities: string[] = [];
   private _lastLuContent: string[] = [];
+  private _curDefinedVariblesInLG: Record<string, any> = {};
+  private _otherDefinedVariblesInLG: Record<string, any> = {};
+  private _mergedVariables: Record<string, any> = {};
   constructor(
     protected readonly connection: IConnection,
     protected readonly getLgResources: (projectId?: string) => ResolverResource[],
     protected readonly memoryResolver?: MemoryResolver,
     protected readonly entitiesResolver?: MemoryResolver
   ) {
     this.documents.listen(this.connection);
-    this.documents.onDidChangeContent((change) => this.validate(change.document));
+    this.documents.onDidChangeContent((change) => {
+      this.validate(change.document);
+      this.updateLGVariables(change.document);
+    });
     this.documents.onDidClose((event) => {
       this.cleanPendingValidation(event.document);
       this.cleanDiagnostics(event.document);
@@ -107,6 +115,8 @@ export class LGServer {
           this.addLGDocument(textDocument, lgOption);
           this.validateLgOption(textDocument, lgOption);
           this.validate(textDocument);
+          this.getOtherLGVariables(lgOption);
+          this.updateMemoryVariables(textDocument);
         }
 
         // update luis entities once user open LG editor
@@ -178,8 +188,8 @@ export class LGServer {
     return items;
   }
 
-  protected updateObject(propertyList: string[]): void {
-    let tempVariable: Record<string, any> = this.memoryVariables;
+  protected updateObject(propertyList: string[], varaibles: Record<string, any>): void {
+    let tempVariable: Record<string, any> = varaibles;
     const antPattern = /\*+/;
     const normalizedAnyPattern = '***';
     for (let property of propertyList) {
@@ -195,12 +205,11 @@ export class LGServer {
     }
   }
 
-  protected updateMemoryVariables(uri: string): void {
+  protected updateMemoryVariables(document: TextDocument): void {
     if (!this.memoryResolver) {
       return;
     }
 
-    const document = this.documents.get(uri);
     if (!document) return;
     const projectId = this.getLGDocument(document)?.projectId;
     if (!projectId) return;
@@ -212,7 +221,7 @@ export class LGServer {
     memoryFileInfo.forEach((variable) => {
       const propertyList = variable.split('.');
       if (propertyList.length >= 1) {
-        this.updateObject(propertyList);
+        this.updateObject(propertyList, this.memoryVariables);
       }
     });
   }
@@ -560,15 +569,16 @@ export class LGServer {
     const range = getRangeAtPosition(document, position);
     const wordAtCurRange = document.getText(range);
     const endWithDot = wordAtCurRange.endsWith('.');
-
-    this.updateMemoryVariables(params.textDocument.uri);
-    const memoryVariblesRootCompletionList = Object.keys(this.memoryVariables).map((e) => {
-      return {
-        label: e.toString(),
-        kind: CompletionItemKind.Property,
-        insertText: e.toString(),
-        documentation: '',
-      };
+    const memoryVariblesRootCompletionList: CompletionItem[] = [];
+    Object.keys(this._mergedVariables).forEach((e) => {
+      if (e.length > 1) {
+        memoryVariblesRootCompletionList.push({
+          label: e.toString(),
+          kind: CompletionItemKind.Property,
+          insertText: e.toString(),
+          documentation: '',
+        });
+      }
     });
 
     if (!wordAtCurRange || !endWithDot) {
@@ -578,7 +588,7 @@ export class LGServer {
     let propertyList = wordAtCurRange.split('.');
     propertyList = propertyList.slice(0, propertyList.length - 1);
 
-    const completionList = this.matchingCompletionProperty(propertyList, this.memoryVariables);
+    const completionList = this.matchingCompletionProperty(propertyList, this._mergedVariables);
 
     return completionList;
   }
@@ -817,6 +827,46 @@ export class LGServer {
     return true;
   }
 
+  protected async getOtherLGVariables(lgOption: LGOption | undefined): Promise<void> {
+    const { fileId, projectId } = lgOption || {};
+    if (projectId && fileId) {
+      const lgTextFiles = projectId ? this.getLgResources(projectId) : [];
+      const lgContents: string[] = [];
+      lgTextFiles.forEach((item) => {
+        if (item.id !== fileId) {
+          lgContents.push(item.content);
+        }
+      });
+
+      const variables = uniq((await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables);
+      this._otherDefinedVariblesInLG = {};
+      variables.forEach((variable) => {
+        const propertyList = variable.split('.');
+        if (propertyList.length >= 1) {
+          this.updateObject(propertyList, this._otherDefinedVariblesInLG);
+        }
+      });
+    }
+  }
+
+  protected async updateLGVariables(document: TextDocument) {
+    const variables = uniq((await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables);
+    this._curDefinedVariblesInLG = {};
+    variables.forEach((variable) => {
+      const propertyList = variable.split('.');
+      if (propertyList.length >= 1) {
+        this.updateObject(propertyList, this._curDefinedVariblesInLG);
+      }
+    });
+
+    this._mergedVariables = merge(
+      {},
+      this.memoryVariables,
+      this._curDefinedVariblesInLG,
+      this._otherDefinedVariblesInLG
+    );
+  }
+
   protected validate(document: TextDocument): void {
     this.cleanPendingValidation(document);
     this.pendingValidationRequests.set(

Composer/packages/tools/language-servers/language-generation/src/lgParser.ts

@@ -9,12 +9,12 @@ import uniqueId from 'lodash/uniqueId';
 import { lgUtil } from '@bfc/indexers';
 import uniq from 'lodash/uniq';
 
-import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils';
+import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, findAllVariables } from './utils';
 
 const isTest = process.env?.NODE_ENV === 'test';
 export interface WorkerMsg {
   id: string;
-  type: 'parse' | 'updateTemplate' | 'extractLuisEntity';
+  type: 'parse' | 'updateTemplate' | 'extractLuisEntity' | 'extractLGVariables';
   error?: any;
   payload?: any;
 }
@@ -44,6 +44,17 @@ class LgParserWithoutWorker {
 
     return { suggestEntities: uniq(suggestEntities) };
   }
+
+  public extractLGVariables(curCbangedFile: string | undefined, lgFiles: string[]) {
+    let result: string[] = [];
+    if (curCbangedFile) {
+      result = findAllVariables(curCbangedFile);
+    } else {
+      result = findAllVariables(lgFiles);
+    }
+
+    return { lgVariables: uniq(result) };
+  }
 }
 
 class LgParserWithWorker {
@@ -90,6 +101,19 @@ class LgParserWithWorker {
     });
   }
 
+  public async extractLGVariables(
+    curCbangedFile: string | undefined,
+    lgFiles: string[]
+  ): Promise<{ lgVariables: string[] }> {
+    const msgId = uniqueId();
+    const msg = { id: msgId, type: 'extractLGVariables', payload: { curCbangedFile, lgFiles } };
+    return new Promise((resolve, reject) => {
+      this.resolves[msgId] = resolve;
+      this.rejects[msgId] = reject;
+      LgParserWithWorker.worker.send(msg);
+    });
+  }
+
   // Handle incoming calculation result
   public handleMsg(msg: WorkerMsg) {
     const { id, error, payload } = msg;

Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts

@@ -6,7 +6,7 @@ import { lgUtil } from '@bfc/indexers';
 import uniq from 'lodash/uniq';
 
 import { WorkerMsg } from './lgParser';
-import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils';
+import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, findAllVariables } from './utils';
 
 process.on('message', async (msg: WorkerMsg) => {
   try {
@@ -44,6 +44,19 @@ process.on('message', async (msg: WorkerMsg) => {
         process.send?.({ id: msg.id, payload: { id, content, templates, allTemplates, diagnostics } });
         break;
       }
+
+      case 'extractLGVariables': {
+        const { curCbangedFile, lgFiles } = msg.payload;
+        let result: string[] = [];
+        if (curCbangedFile) {
+          result = findAllVariables(curCbangedFile);
+        } else {
+          result = findAllVariables(lgFiles);
+        }
+
+        process.send?.({ id: msg.id, payload: { lgVariables: result } });
+        break;
+      }
     }
   } catch (error) {
     process.send?.({ id: msg.id, error });

Composer/packages/tools/language-servers/language-generation/src/utils.ts

@@ -7,6 +7,9 @@ import { DiagnosticSeverity as LGDiagnosticSeverity } from 'botbuilder-lg';
 import { Diagnostic as BFDiagnostic, LgFile } from '@bfc/shared';
 import { parser } from '@microsoft/bf-lu/lib/parser';
 import { offsetRange } from '@bfc/indexers';
+import { LGResource, Templates } from 'botbuilder-lg';
+import { Expression } from 'adaptive-expressions';
+import uniq from 'lodash/uniq';
 
 const { parseFile } = parser;
 
@@ -224,3 +227,78 @@ export function getLineByIndex(document: TextDocument, line: number) {
 
   return document.getText().split(/\r?\n/g)[line];
 }
+
+function findExpr(pst: any, result: string[]): void {
+  const exprRegex = /\$\{.*\}/;
+  if (pst.childCount === 0) {
+    if (exprRegex.test(pst.text)) {
+      result.push(pst.text.substr(2, pst.text.length - 3));
+    }
+  } else {
+    const count = pst.childCount;
+    for (let i = 0; i < count; i++) {
+      const child = pst.getChild(i);
+      findExpr(child, result);
+    }
+  }
+}
+
+function findAllExprs(contents: string | string[]): string[] {
+  const result = [];
+  if (typeof contents === 'string') {
+    try {
+      const templates = Templates.parseResource(new LGResource('id', 'name', contents));
+      templates.allTemplates.forEach((t) => {
+        const parseTree = t.templateBodyParseTree;
+        findExpr(parseTree, result);
+      });
+    } catch (e) {
+      // do nothing
+    }
+  } else {
+    for (const lg of contents) {
+      try {
+        const templates = Templates.parseResource(new LGResource('id', 'name', lg));
+        templates.allTemplates.forEach((t) => {
+          const parseTree = t.templateBodyParseTree;
+          findExpr(parseTree, result);
+        });
+      } catch (e) {
+        // do nothing
+      }
+    }
+  }
+
+  return uniq(result);
+}
+
+function findVariables(expr: Expression, result: string[]) {
+  if (expr.type === 'Accessor') {
+    result.push(expr.toString());
+  } else {
+    if (expr.children.length >= 1) {
+      for (const child of expr.children) {
+        if (child.type === 'Accessor') {
+          result.push(child.toString());
+        } else {
+          findVariables(child, result);
+        }
+      }
+    }
+  }
+}
+
+export function findAllVariables(contents: string | string[]) {
+  const exprs = findAllExprs(contents);
+  const result = [];
+  for (const expr of exprs) {
+    try {
+      const exp = Expression.parse(expr);
+      findVariables(exp, result);
+    } catch (e) {
+      // do nothing
+    }
+  }
+
+  return uniq(result);
+}