From 803b07033ac7d966c02606eabce3f705bbfb14cf Mon Sep 17 00:00:00 2001 From: Shuai Wang Date: Tue, 9 Mar 2021 19:21:57 +0800 Subject: [PATCH 1/8] init --- .../language-generation/src/LGServer.ts | 21 +++++++++++++ .../language-generation/src/lgParser.ts | 30 +++++++++++++++++-- .../language-generation/src/lgWorker.ts | 17 ++++++++++- .../language-generation/src/utils.ts | 30 +++++++++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index 6746e47725..28bdc6ff0b 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -21,6 +21,7 @@ import { FoldingRange, } from 'vscode-languageserver-protocol'; import get from 'lodash/get'; +import uniq from 'lodash/uniq'; import isEqual from 'lodash/isEqual'; import { filterTemplateDiagnostics, isValid, lgUtil } from '@bfc/indexers'; import { MemoryResolver, ResolverResource, LgFile } from '@bfc/shared'; @@ -55,6 +56,7 @@ export class LGServer { private _lgParser = new LgParser(); private _luisEntities: string[] = []; private _lastLuContent: string[] = []; + private _userDefinedVariblesInLG: string[] = []; constructor( protected readonly connection: IConnection, protected readonly getLgResources: (projectId?: string) => ResolverResource[], @@ -193,6 +195,8 @@ export class LGServer { tempVariable = tempVariable[property]; } } + + console.log(this.memoryVariables); } protected updateMemoryVariables(uri: string): void { @@ -215,6 +219,14 @@ export class LGServer { this.updateObject(propertyList); } }); + + console.log(this._userDefinedVariblesInLG); + this._userDefinedVariblesInLG.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList); + } + }); } protected async validateLgOption(document: TextDocument, lgOption?: LGOption) { @@ -239,6 +251,8 @@ export class LGServer { const content = this.documents.get(uri)?.getText() || ''; // if inline mode, composite local with server resolved file. const lgTextFiles = projectId ? this.getLgResources(projectId) : []; + const lgContents: string[] = lgTextFiles.map((e) => e.content); + this._userDefinedVariblesInLG = (await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables; if (fileId && templateId) { const lgTextFile = lgTextFiles.find((item) => item.id === fileId); if (lgTextFile) { @@ -780,6 +794,13 @@ export class LGServer { protected validate(document: TextDocument): void { this.cleanPendingValidation(document); + setTimeout(async () => { + this._userDefinedVariblesInLG = uniq( + this._userDefinedVariblesInLG.concat( + (await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables + ) + ); + }); this.pendingValidationRequests.set( document.uri, setTimeout(async () => { diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts index b0d099da43..2a1e78460e 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts @@ -9,12 +9,12 @@ import uniqueId from 'lodash/uniqueId'; import { lgUtil } from '@bfc/indexers'; import uniq from 'lodash/uniq'; -import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils'; +import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, extractVaribles } from './utils'; const isTest = process.env?.NODE_ENV === 'test'; export interface WorkerMsg { id: string; - type: 'parse' | 'updateTemplate' | 'extractLuisEntity'; + type: 'parse' | 'updateTemplate' | 'extractLuisEntity' | 'extractLGVariables'; error?: any; payload?: any; } @@ -44,6 +44,19 @@ class LgParserWithoutWorker { return { suggestEntities: uniq(suggestEntities) }; } + + public async extractLGVariables(curCbangedFile: string | undefined, lgFiles: string[]) { + let result: string[] = []; + if (curCbangedFile) { + result = extractVaribles(curCbangedFile); + } else { + for (const file of lgFiles) { + result = result.concat(extractVaribles(file)); + } + } + + return { lgVariables: uniq(result) }; + } } class LgParserWithWorker { @@ -90,6 +103,19 @@ class LgParserWithWorker { }); } + public async extractLGVariables( + curCbangedFile: string | undefined, + lgFiles: string[] + ): Promise<{ lgVariables: string[] }> { + const msgId = uniqueId(); + const msg = { id: msgId, type: 'extractLGVariables', payload: { curCbangedFile, lgFiles } }; + return new Promise((resolve, reject) => { + this.resolves[msgId] = resolve; + this.rejects[msgId] = reject; + LgParserWithWorker.worker.send(msg); + }); + } + // Handle incoming calculation result public handleMsg(msg: WorkerMsg) { const { id, error, payload } = msg; diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts index fb3d8faa72..179ed2eb6c 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts @@ -6,7 +6,7 @@ import { lgUtil } from '@bfc/indexers'; import uniq from 'lodash/uniq'; import { WorkerMsg } from './lgParser'; -import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils'; +import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, extractVaribles } from './utils'; process.on('message', async (msg: WorkerMsg) => { try { @@ -44,6 +44,21 @@ process.on('message', async (msg: WorkerMsg) => { process.send?.({ id: msg.id, payload: { id, content, templates, allTemplates, diagnostics } }); break; } + + case 'extractLGVariables': { + const { curCbangedFile, lgFiles } = msg.payload; + let result: string[] = []; + if (curCbangedFile) { + result = extractVaribles(curCbangedFile); + } else { + for (const file of lgFiles) { + result = result.concat(extractVaribles(file)); + } + } + + process.send?.({ id: msg.id, payload: { lgVariables: result } }); + break; + } } } catch (error) { process.send?.({ id: msg.id, error }); diff --git a/Composer/packages/tools/language-servers/language-generation/src/utils.ts b/Composer/packages/tools/language-servers/language-generation/src/utils.ts index da8961b2e8..bea4f9edd0 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/utils.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/utils.ts @@ -224,3 +224,33 @@ export function getLineByIndex(document: TextDocument, line: number) { return document.getText().split(/\r?\n/g)[line]; } + +export function extractVaribles(content: string): string[] { + const lines = content.split(/\r?\n/g); + const keyValueRegex = /\s*[a-zA-Z]+\s*=.+/; + const exprRegex = /\$\{.+\}/g; + const varabileRegex = /[a-zA-Z].[a-zA-Z0-9.]+/; + let varibles: string[] = []; + for (const line of lines) { + if (line.trim().startsWith('-') || keyValueRegex.test(line)) { + const exprs = line.match(exprRegex); + if (exprs) { + for (const expr of exprs) { + //trim the starting ${ and ending } + const body = expr.substr(2, expr.length - 3); + const removed = body + .replace(/[a-zA-Z0-9.]+\(/, '') + .replace(/"[^"]*"/, '') + .replace(/'[^']*'/, '') + .replace(')', ''); + const localVaribles = removed.match(varabileRegex); + if (localVaribles) { + varibles = varibles.concat(localVaribles); + } + } + } + } + } + + return varibles; +} From 6aedef419fe3f032886bd43e46b1f86e44986201 Mon Sep 17 00:00:00 2001 From: Shuai Wang Date: Tue, 9 Mar 2021 21:16:47 +0800 Subject: [PATCH 2/8] cache variabkes defined in LG files --- .../language-generation/src/LGServer.ts | 29 +++++++++++++------ .../language-generation/src/utils.ts | 3 +- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index 28bdc6ff0b..f48005eaec 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -56,7 +56,8 @@ export class LGServer { private _lgParser = new LgParser(); private _luisEntities: string[] = []; private _lastLuContent: string[] = []; - private _userDefinedVariblesInLG: string[] = []; + private _curDefinedVariblesInLG: string[] = []; + private _otherDefinedVariblesInLG: string[] = []; constructor( protected readonly connection: IConnection, protected readonly getLgResources: (projectId?: string) => ResolverResource[], @@ -213,6 +214,7 @@ export class LGServer { return; } + this.memoryVariables = {}; memoryFileInfo.forEach((variable) => { const propertyList = variable.split('.'); if (propertyList.length >= 1) { @@ -220,8 +222,14 @@ export class LGServer { } }); - console.log(this._userDefinedVariblesInLG); - this._userDefinedVariblesInLG.forEach((variable) => { + this._curDefinedVariblesInLG.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList); + } + }); + + this._otherDefinedVariblesInLG.forEach((variable) => { const propertyList = variable.split('.'); if (propertyList.length >= 1) { this.updateObject(propertyList); @@ -251,8 +259,13 @@ export class LGServer { const content = this.documents.get(uri)?.getText() || ''; // if inline mode, composite local with server resolved file. const lgTextFiles = projectId ? this.getLgResources(projectId) : []; - const lgContents: string[] = lgTextFiles.map((e) => e.content); - this._userDefinedVariblesInLG = (await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables; + const lgContents: string[] = []; + lgTextFiles.forEach((item) => { + if (item.id !== fileId) { + lgContents.push(item.content); + } + }); + this._otherDefinedVariblesInLG = (await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables; if (fileId && templateId) { const lgTextFile = lgTextFiles.find((item) => item.id === fileId); if (lgTextFile) { @@ -795,10 +808,8 @@ export class LGServer { protected validate(document: TextDocument): void { this.cleanPendingValidation(document); setTimeout(async () => { - this._userDefinedVariblesInLG = uniq( - this._userDefinedVariblesInLG.concat( - (await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables - ) + this._curDefinedVariblesInLG = uniq( + (await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables ); }); this.pendingValidationRequests.set( diff --git a/Composer/packages/tools/language-servers/language-generation/src/utils.ts b/Composer/packages/tools/language-servers/language-generation/src/utils.ts index bea4f9edd0..6c6823e8e0 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/utils.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/utils.ts @@ -229,7 +229,8 @@ export function extractVaribles(content: string): string[] { const lines = content.split(/\r?\n/g); const keyValueRegex = /\s*[a-zA-Z]+\s*=.+/; const exprRegex = /\$\{.+\}/g; - const varabileRegex = /[a-zA-Z].[a-zA-Z0-9.]+/; + // eslint-disable-next-line security/detect-unsafe-regex + const varabileRegex = /[a-zA-Z]+\.[a-zA-Z0-9]+(\.[a-zA-Z0-9.]+)?/g; let varibles: string[] = []; for (const line of lines) { if (line.trim().startsWith('-') || keyValueRegex.test(line)) { From 28f854f51db4b2bc36c34b5c283284786644d85a Mon Sep 17 00:00:00 2001 From: Shuai Wang Date: Wed, 10 Mar 2021 14:46:20 +0800 Subject: [PATCH 3/8] reduce calculation of variables --- .../language-generation/src/LGServer.ts | 100 +++++++++++------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index f48005eaec..de0ffcf2e2 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -22,6 +22,7 @@ import { } from 'vscode-languageserver-protocol'; import get from 'lodash/get'; import uniq from 'lodash/uniq'; +import merge from 'lodash/merge'; import isEqual from 'lodash/isEqual'; import { filterTemplateDiagnostics, isValid, lgUtil } from '@bfc/indexers'; import { MemoryResolver, ResolverResource, LgFile } from '@bfc/shared'; @@ -56,8 +57,9 @@ export class LGServer { private _lgParser = new LgParser(); private _luisEntities: string[] = []; private _lastLuContent: string[] = []; - private _curDefinedVariblesInLG: string[] = []; - private _otherDefinedVariblesInLG: string[] = []; + private _curDefinedVariblesInLG: Record = {}; + private _otherDefinedVariblesInLG: Record = {}; + private _mergedVariables: Record = {}; constructor( protected readonly connection: IConnection, protected readonly getLgResources: (projectId?: string) => ResolverResource[], @@ -65,7 +67,10 @@ export class LGServer { protected readonly entitiesResolver?: MemoryResolver ) { this.documents.listen(this.connection); - this.documents.onDidChangeContent((change) => this.validate(change.document)); + this.documents.onDidChangeContent((change) => { + this.validate(change.document); + this.updateLGVariables(change.document); + }); this.documents.onDidClose((event) => { this.cleanPendingValidation(event.document); this.cleanDiagnostics(event.document); @@ -110,6 +115,8 @@ export class LGServer { this.addLGDocument(textDocument, lgOption); this.validateLgOption(textDocument, lgOption); this.validate(textDocument); + this.getOtherLGVariables(lgOption); + this.updateMemoryVariables(textDocument); } // update luis entities once user open LG editor @@ -181,8 +188,8 @@ export class LGServer { return items; } - protected updateObject(propertyList: string[]): void { - let tempVariable: Record = this.memoryVariables; + protected updateObject(propertyList: string[], varaibles: Record): void { + let tempVariable: Record = varaibles; const antPattern = /\*+/; const normalizedAnyPattern = '***'; for (let property of propertyList) { @@ -196,16 +203,13 @@ export class LGServer { tempVariable = tempVariable[property]; } } - - console.log(this.memoryVariables); } - protected updateMemoryVariables(uri: string): void { + protected updateMemoryVariables(document: TextDocument): void { if (!this.memoryResolver) { return; } - const document = this.documents.get(uri); if (!document) return; const projectId = this.getLGDocument(document)?.projectId; if (!projectId) return; @@ -214,25 +218,10 @@ export class LGServer { return; } - this.memoryVariables = {}; memoryFileInfo.forEach((variable) => { const propertyList = variable.split('.'); if (propertyList.length >= 1) { - this.updateObject(propertyList); - } - }); - - this._curDefinedVariblesInLG.forEach((variable) => { - const propertyList = variable.split('.'); - if (propertyList.length >= 1) { - this.updateObject(propertyList); - } - }); - - this._otherDefinedVariblesInLG.forEach((variable) => { - const propertyList = variable.split('.'); - if (propertyList.length >= 1) { - this.updateObject(propertyList); + this.updateObject(propertyList, this.memoryVariables); } }); } @@ -258,14 +247,8 @@ export class LGServer { const index = async (): Promise => { const content = this.documents.get(uri)?.getText() || ''; // if inline mode, composite local with server resolved file. + console.log('projectId', projectId); const lgTextFiles = projectId ? this.getLgResources(projectId) : []; - const lgContents: string[] = []; - lgTextFiles.forEach((item) => { - if (item.id !== fileId) { - lgContents.push(item.content); - } - }); - this._otherDefinedVariblesInLG = (await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables; if (fileId && templateId) { const lgTextFile = lgTextFiles.find((item) => item.id === fileId); if (lgTextFile) { @@ -556,9 +539,7 @@ export class LGServer { const range = getRangeAtPosition(document, position); const wordAtCurRange = document.getText(range); const endWithDot = wordAtCurRange.endsWith('.'); - - this.updateMemoryVariables(params.textDocument.uri); - const memoryVariblesRootCompletionList = Object.keys(this.memoryVariables).map((e) => { + const memoryVariblesRootCompletionList = Object.keys(this._mergedVariables).map((e) => { return { label: e.toString(), kind: CompletionItemKind.Property, @@ -574,7 +555,7 @@ export class LGServer { let propertyList = wordAtCurRange.split('.'); propertyList = propertyList.slice(0, propertyList.length - 1); - const completionList = this.matchingCompletionProperty(propertyList, this.memoryVariables); + const completionList = this.matchingCompletionProperty(propertyList, this._mergedVariables); return completionList; } @@ -805,13 +786,52 @@ export class LGServer { return true; } - protected validate(document: TextDocument): void { - this.cleanPendingValidation(document); + protected getOtherLGVariables(lgOption: LGOption | undefined): void { + const { fileId, projectId } = lgOption || {}; + if (projectId && fileId) { + const lgTextFiles = projectId ? this.getLgResources(projectId) : []; + const lgContents: string[] = []; + lgTextFiles.forEach((item) => { + if (item.id !== fileId) { + lgContents.push(item.content); + } + }); + + setTimeout(async () => { + const variables = uniq((await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables); + this._otherDefinedVariblesInLG = {}; + variables.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList, this._otherDefinedVariblesInLG); + } + }); + }); + } + } + + protected updateLGVariables(document: TextDocument) { setTimeout(async () => { - this._curDefinedVariblesInLG = uniq( - (await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables + const variables = uniq((await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables); + this._curDefinedVariblesInLG = {}; + variables.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList, this._curDefinedVariblesInLG); + } + }); + + this._mergedVariables = merge( + {}, + this.memoryVariables, + this._curDefinedVariblesInLG, + this._otherDefinedVariblesInLG ); }); + } + + protected validate(document: TextDocument): void { + this.cleanPendingValidation(document); this.pendingValidationRequests.set( document.uri, setTimeout(async () => { From 86878cb75f6bbe34d6384cb33a4425ce685efb28 Mon Sep 17 00:00:00 2001 From: Shuai Wang Date: Wed, 10 Mar 2021 14:57:02 +0800 Subject: [PATCH 4/8] remove console log --- .../tools/language-servers/language-generation/src/LGServer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index c0dd731eaa..f52edd1766 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -247,7 +247,6 @@ export class LGServer { const index = async (): Promise => { const content = this.documents.get(uri)?.getText() || ''; // if inline mode, composite local with server resolved file. - console.log('projectId', projectId); const lgTextFiles = projectId ? this.getLgResources(projectId) : []; if (fileId && templateId) { const lgTextFile = lgTextFiles.find((item) => item.id === fileId); From fed475b6e8c8c4775c8ffdacc8a39739e68e1fc6 Mon Sep 17 00:00:00 2001 From: Shuai Wang Date: Thu, 11 Mar 2021 15:32:06 +0800 Subject: [PATCH 5/8] add find expression by lg parser --- .../language-generation/src/utils.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Composer/packages/tools/language-servers/language-generation/src/utils.ts b/Composer/packages/tools/language-servers/language-generation/src/utils.ts index 4b0291da2f..aca0e2143c 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/utils.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/utils.ts @@ -7,6 +7,7 @@ import { DiagnosticSeverity as LGDiagnosticSeverity } from 'botbuilder-lg'; import { Diagnostic as BFDiagnostic, LgFile } from '@bfc/shared'; import { parser } from '@microsoft/bf-lu/lib/parser'; import { offsetRange } from '@bfc/indexers'; +import { LGResource, Templates } from 'botbuilder-lg'; const { parseFile } = parser; @@ -255,3 +256,29 @@ export function extractVaribles(content: string): string[] { return varibles; } + +function findExpr(pst: any, result: string[]): void { + const exprRegex = /\$\{.*\}/; + if (pst.childCount === 0) { + if (exprRegex.test(pst.text)) { + result.push(pst.text.substr(2, pst.text.length - 3)); + } + } else { + const count = pst.childCount; + for (let i = 0; i < count; i++) { + const child = pst.getChild(i); + findExpr(child, result); + } + } +} + +function findAllExprs(content: string): string[] { + const result = []; + const templates = Templates.parseResource(new LGResource('id', 'name', content)); + templates.allTemplates.forEach((t) => { + const parseTree = t.templateBodyParseTree; + findExpr(parseTree, result); + }); + + return result; +} From 0e51ad16d6556aaaf24571ee5b124c14ff0c9e24 Mon Sep 17 00:00:00 2001 From: cosmicshuai Date: Thu, 11 Mar 2021 16:53:42 +0800 Subject: [PATCH 6/8] change find variables --- .../language-generation/package.json | 1 + .../language-generation/src/lgParser.ts | 8 +-- .../language-generation/src/lgWorker.ts | 10 +-- .../language-generation/src/utils.ts | 66 ++++++++----------- 4 files changed, 37 insertions(+), 48 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/package.json b/Composer/packages/tools/language-servers/language-generation/package.json index 1e9fc1217a..6fb78349a4 100644 --- a/Composer/packages/tools/language-servers/language-generation/package.json +++ b/Composer/packages/tools/language-servers/language-generation/package.json @@ -18,6 +18,7 @@ "@bfc/built-in-functions": "*", "@bfc/indexers": "*", "botbuilder-lg": "4.12.0-rc1", + "adaptive-expressions": "4.12.0-rc1", "vscode-languageserver": "^5.3.0-next" }, "devDependencies": { diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts index 2a1e78460e..6549a1d1f8 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts @@ -9,7 +9,7 @@ import uniqueId from 'lodash/uniqueId'; import { lgUtil } from '@bfc/indexers'; import uniq from 'lodash/uniq'; -import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, extractVaribles } from './utils'; +import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, findAllVariables } from './utils'; const isTest = process.env?.NODE_ENV === 'test'; export interface WorkerMsg { @@ -48,11 +48,9 @@ class LgParserWithoutWorker { public async extractLGVariables(curCbangedFile: string | undefined, lgFiles: string[]) { let result: string[] = []; if (curCbangedFile) { - result = extractVaribles(curCbangedFile); + result = findAllVariables(curCbangedFile); } else { - for (const file of lgFiles) { - result = result.concat(extractVaribles(file)); - } + result = findAllVariables(lgFiles); } return { lgVariables: uniq(result) }; diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts index 179ed2eb6c..b812788d33 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts @@ -5,8 +5,10 @@ import { lgImportResolverGenerator } from '@bfc/shared'; import { lgUtil } from '@bfc/indexers'; import uniq from 'lodash/uniq'; +import { findAllExprs } from '../lib/utils'; + import { WorkerMsg } from './lgParser'; -import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, extractVaribles } from './utils'; +import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils'; process.on('message', async (msg: WorkerMsg) => { try { @@ -49,11 +51,9 @@ process.on('message', async (msg: WorkerMsg) => { const { curCbangedFile, lgFiles } = msg.payload; let result: string[] = []; if (curCbangedFile) { - result = extractVaribles(curCbangedFile); + result = findAllExprs(curCbangedFile); } else { - for (const file of lgFiles) { - result = result.concat(extractVaribles(file)); - } + result = findAllExprs(lgFiles); } process.send?.({ id: msg.id, payload: { lgVariables: result } }); diff --git a/Composer/packages/tools/language-servers/language-generation/src/utils.ts b/Composer/packages/tools/language-servers/language-generation/src/utils.ts index aca0e2143c..2e631decc1 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/utils.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/utils.ts @@ -8,6 +8,7 @@ import { Diagnostic as BFDiagnostic, LgFile } from '@bfc/shared'; import { parser } from '@microsoft/bf-lu/lib/parser'; import { offsetRange } from '@bfc/indexers'; import { LGResource, Templates } from 'botbuilder-lg'; +import uniq from 'lodash/uniq'; const { parseFile } = parser; @@ -226,37 +227,6 @@ export function getLineByIndex(document: TextDocument, line: number) { return document.getText().split(/\r?\n/g)[line]; } -export function extractVaribles(content: string): string[] { - const lines = content.split(/\r?\n/g); - const keyValueRegex = /\s*[a-zA-Z]+\s*=.+/; - const exprRegex = /\$\{.+\}/g; - // eslint-disable-next-line security/detect-unsafe-regex - const varabileRegex = /[a-zA-Z]+\.[a-zA-Z0-9]+(\.[a-zA-Z0-9.]+)?/g; - let varibles: string[] = []; - for (const line of lines) { - if (line.trim().startsWith('-') || keyValueRegex.test(line)) { - const exprs = line.match(exprRegex); - if (exprs) { - for (const expr of exprs) { - //trim the starting ${ and ending } - const body = expr.substr(2, expr.length - 3); - const removed = body - .replace(/[a-zA-Z0-9.]+\(/, '') - .replace(/"[^"]*"/, '') - .replace(/'[^']*'/, '') - .replace(')', ''); - const localVaribles = removed.match(varabileRegex); - if (localVaribles) { - varibles = varibles.concat(localVaribles); - } - } - } - } - } - - return varibles; -} - function findExpr(pst: any, result: string[]): void { const exprRegex = /\$\{.*\}/; if (pst.childCount === 0) { @@ -272,13 +242,33 @@ function findExpr(pst: any, result: string[]): void { } } -function findAllExprs(content: string): string[] { +function findAllExprs(contents: string | string[]): string[] { const result = []; - const templates = Templates.parseResource(new LGResource('id', 'name', content)); - templates.allTemplates.forEach((t) => { - const parseTree = t.templateBodyParseTree; - findExpr(parseTree, result); - }); + if (typeof contents === 'string') { + const templates = Templates.parseResource(new LGResource('id', 'name', contents)); + templates.allTemplates.forEach((t) => { + const parseTree = t.templateBodyParseTree; + findExpr(parseTree, result); + }); + } else { + for (const lg of contents) { + const templates = Templates.parseResource(new LGResource('id', 'name', lg)); + templates.allTemplates.forEach((t) => { + const parseTree = t.templateBodyParseTree; + findExpr(parseTree, result); + }); + } + } + + return uniq(result); +} + +export function findAllVariables(contents: string | string[]) { + const exprs = findAllExprs(contents); + const result = []; + for (const expr of exprs) { + findExpr(expr, result); + } - return result; + return uniq(result); } From 9afa0bc9f30611f0d055bd31a577cb52b31e6282 Mon Sep 17 00:00:00 2001 From: cosmicshuai Date: Thu, 11 Mar 2021 20:00:45 +0800 Subject: [PATCH 7/8] fix some issues --- .../language-generation/src/LGServer.ts | 49 ++++++++++--------- .../language-generation/src/lgParser.ts | 2 +- .../language-generation/src/lgWorker.ts | 8 ++- .../language-generation/src/utils.ts | 48 ++++++++++++++---- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index f52edd1766..af0abc8a48 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -569,13 +569,16 @@ export class LGServer { const range = getRangeAtPosition(document, position); const wordAtCurRange = document.getText(range); const endWithDot = wordAtCurRange.endsWith('.'); - const memoryVariblesRootCompletionList = Object.keys(this._mergedVariables).map((e) => { - return { - label: e.toString(), - kind: CompletionItemKind.Property, - insertText: e.toString(), - documentation: '', - }; + const memoryVariblesRootCompletionList: CompletionItem[] = []; + Object.keys(this._mergedVariables).forEach((e) => { + if (e.length > 1) { + memoryVariblesRootCompletionList.push({ + label: e.toString(), + kind: CompletionItemKind.Property, + insertText: e.toString(), + documentation: '', + }); + } }); if (!wordAtCurRange || !endWithDot) { @@ -848,24 +851,22 @@ export class LGServer { } } - protected updateLGVariables(document: TextDocument) { - setTimeout(async () => { - const variables = uniq((await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables); - this._curDefinedVariblesInLG = {}; - variables.forEach((variable) => { - const propertyList = variable.split('.'); - if (propertyList.length >= 1) { - this.updateObject(propertyList, this._curDefinedVariblesInLG); - } - }); - - this._mergedVariables = merge( - {}, - this.memoryVariables, - this._curDefinedVariblesInLG, - this._otherDefinedVariblesInLG - ); + protected async updateLGVariables(document: TextDocument) { + const variables = uniq((await this._lgParser.extractLGVariables(document.getText(), [])).lgVariables); + this._curDefinedVariblesInLG = {}; + variables.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList, this._curDefinedVariblesInLG); + } }); + + this._mergedVariables = merge( + {}, + this.memoryVariables, + this._curDefinedVariblesInLG, + this._otherDefinedVariblesInLG + ); } protected validate(document: TextDocument): void { diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts index 6549a1d1f8..06ee5c5f8e 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgParser.ts @@ -45,7 +45,7 @@ class LgParserWithoutWorker { return { suggestEntities: uniq(suggestEntities) }; } - public async extractLGVariables(curCbangedFile: string | undefined, lgFiles: string[]) { + public extractLGVariables(curCbangedFile: string | undefined, lgFiles: string[]) { let result: string[] = []; if (curCbangedFile) { result = findAllVariables(curCbangedFile); diff --git a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts index b812788d33..85e518e72c 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/lgWorker.ts @@ -5,10 +5,8 @@ import { lgImportResolverGenerator } from '@bfc/shared'; import { lgUtil } from '@bfc/indexers'; import uniq from 'lodash/uniq'; -import { findAllExprs } from '../lib/utils'; - import { WorkerMsg } from './lgParser'; -import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes } from './utils'; +import { getSuggestionEntities, extractLUISContent, suggestionAllEntityTypes, findAllVariables } from './utils'; process.on('message', async (msg: WorkerMsg) => { try { @@ -51,9 +49,9 @@ process.on('message', async (msg: WorkerMsg) => { const { curCbangedFile, lgFiles } = msg.payload; let result: string[] = []; if (curCbangedFile) { - result = findAllExprs(curCbangedFile); + result = findAllVariables(curCbangedFile); } else { - result = findAllExprs(lgFiles); + result = findAllVariables(lgFiles); } process.send?.({ id: msg.id, payload: { lgVariables: result } }); diff --git a/Composer/packages/tools/language-servers/language-generation/src/utils.ts b/Composer/packages/tools/language-servers/language-generation/src/utils.ts index 2e631decc1..089f3783c1 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/utils.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/utils.ts @@ -8,6 +8,7 @@ import { Diagnostic as BFDiagnostic, LgFile } from '@bfc/shared'; import { parser } from '@microsoft/bf-lu/lib/parser'; import { offsetRange } from '@bfc/indexers'; import { LGResource, Templates } from 'botbuilder-lg'; +import { Expression } from 'adaptive-expressions'; import uniq from 'lodash/uniq'; const { parseFile } = parser; @@ -245,29 +246,58 @@ function findExpr(pst: any, result: string[]): void { function findAllExprs(contents: string | string[]): string[] { const result = []; if (typeof contents === 'string') { - const templates = Templates.parseResource(new LGResource('id', 'name', contents)); - templates.allTemplates.forEach((t) => { - const parseTree = t.templateBodyParseTree; - findExpr(parseTree, result); - }); - } else { - for (const lg of contents) { - const templates = Templates.parseResource(new LGResource('id', 'name', lg)); + try { + const templates = Templates.parseResource(new LGResource('id', 'name', contents)); templates.allTemplates.forEach((t) => { const parseTree = t.templateBodyParseTree; findExpr(parseTree, result); }); + } catch (e) { + // do nothing + } + } else { + for (const lg of contents) { + try { + const templates = Templates.parseResource(new LGResource('id', 'name', lg)); + templates.allTemplates.forEach((t) => { + const parseTree = t.templateBodyParseTree; + findExpr(parseTree, result); + }); + } catch (e) { + // do nothing + } } } return uniq(result); } +function findVariables(expr: Expression, result: string[]) { + if (expr.type === 'Accessor') { + result.push(expr.toString()); + } else { + if (expr.children.length >= 1) { + for (const child of expr.children) { + if (child.type === 'Accessor') { + result.push(child.toString()); + } else { + findVariables(child, result); + } + } + } + } +} + export function findAllVariables(contents: string | string[]) { const exprs = findAllExprs(contents); const result = []; for (const expr of exprs) { - findExpr(expr, result); + try { + const exp = Expression.parse(expr); + findVariables(exp, result); + } catch (e) { + // do nothing + } } return uniq(result); From 6a3a247655a80cf004822c134464a0fab72db611 Mon Sep 17 00:00:00 2001 From: cosmicshuai Date: Fri, 12 Mar 2021 10:13:52 +0800 Subject: [PATCH 8/8] remove a setimeout func --- .../language-generation/src/LGServer.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts index af0abc8a48..12c78be9ae 100644 --- a/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts +++ b/Composer/packages/tools/language-servers/language-generation/src/LGServer.ts @@ -827,7 +827,7 @@ export class LGServer { return true; } - protected getOtherLGVariables(lgOption: LGOption | undefined): void { + protected async getOtherLGVariables(lgOption: LGOption | undefined): Promise { const { fileId, projectId } = lgOption || {}; if (projectId && fileId) { const lgTextFiles = projectId ? this.getLgResources(projectId) : []; @@ -838,15 +838,13 @@ export class LGServer { } }); - setTimeout(async () => { - const variables = uniq((await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables); - this._otherDefinedVariblesInLG = {}; - variables.forEach((variable) => { - const propertyList = variable.split('.'); - if (propertyList.length >= 1) { - this.updateObject(propertyList, this._otherDefinedVariblesInLG); - } - }); + const variables = uniq((await this._lgParser.extractLGVariables(undefined, lgContents)).lgVariables); + this._otherDefinedVariblesInLG = {}; + variables.forEach((variable) => { + const propertyList = variable.split('.'); + if (propertyList.length >= 1) { + this.updateObject(propertyList, this._otherDefinedVariblesInLG); + } }); } }