kie-issues#1178: On the DMN Editor's Boxed Expression Editor, FEEL identifiers with spaces should be treated as non-breaking spaces.

packages/dmn-feel-antlr4-parser/src/parser/grammar/ParserHelper.ts

@@ -186,7 +186,11 @@ export class ParserHelper {
     const startLine = _n1.start.line - 1;
     const endLine = _n1.stop?.line !== undefined ? _n1.stop.line - 1 : startLine;
 
-    const variableName = name.replaceAll("\n", "");
+    // Replace line-breaks and multiple blank-spaces, since it is considered valid in variables names.
+    // Notice that line-brakes behave exactly like blank-spaces, that's why we're replacing them to blank-spaces.
+    // The Regex is to replace all concatenated blank-spaces to a single one. For example:
+    // "a           b"  becomes "a b", because that's how it is handled in the DMN runner.
+    const variableName = name.replaceAll("\r\n", " ").replaceAll("\n", " ").replace(/\s\s+/g, " ");
     if (this.currentScope?.getChildScopes().has(variableName)) {
       this.variables.push(
         new FeelVariable(start, length, startLine, endLine, FeelSyntacticSymbolNature.GlobalVariable, variableName)

packages/feel-input-component/jest.config.js

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { config, babelTransform, typescriptTransform } = require("@kie-tools/jest-base/jest.config");
+
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  ...config,
+  testEnvironment: "node",
+  transform: {
+    ...babelTransform,
+    ...typescriptTransform,
+  },
+};

packages/feel-input-component/package.json

@@ -8,12 +8,13 @@
   "main": "dist/index.js",
   "scripts": {
     "build:dev": "rimraf dist && pnpm copy:css && tsc -p tsconfig.json",
-    "build:prod": "rimraf dist && pnpm copy:css && pnpm lint && tsc -p tsconfig.json",
+    "build:prod": "rimraf dist && pnpm copy:css && pnpm lint && tsc -p tsconfig.json && pnpm test",
     "build:showcase": "rimraf ./dist-dev && webpack -c ./showcase/webpack.config.js --env prod",
     "copy:css": "copyfiles -u 1 \"src/**/*.{sass,scss,css}\" dist/",
     "deploy": "gh-pages -d dist-dev",
     "lint": "run-script-if --bool \"$(build-env linters.run)\" --then \"kie-tools--eslint ./src\"",
-    "start": "webpack serve -c ./showcase/webpack.config.js --host 0.0.0.0 --env dev"
+    "start": "webpack serve -c ./showcase/webpack.config.js --host 0.0.0.0 --env dev",
+    "test": "rimraf dist-tests && run-script-if --ignore-errors \"$(build-env tests.ignoreFailures)\" --bool \"$(build-env tests.run)\" --then \"jest --silent --verbose --passWithNoTests\""
   },
   "dependencies": {
     "@kie-tools-core/i18n": "workspace:*",
@@ -32,14 +33,21 @@
     "@babel/preset-react": "^7.16.0",
     "@kie-tools-core/webpack-base": "workspace:*",
     "@kie-tools/eslint": "workspace:*",
+    "@kie-tools/jest-base": "workspace:*",
     "@kie-tools/root-env": "workspace:*",
     "@kie-tools/tsconfig": "workspace:*",
+    "@types/jest": "^29.5.12",
+    "@types/jest-when": "^3.5.5",
     "@types/react": "^17.0.6",
     "@types/react-dom": "^17.0.5",
     "copy-webpack-plugin": "^11.0.0",
     "copyfiles": "^2.4.1",
     "file-loader": "^6.2.0",
+    "jest": "^29.7.0",
+    "jest-junit": "^16.0.0",
+    "jest-when": "^3.6.0",
     "rimraf": "^3.0.2",
+    "ts-jest": "^29.1.5",
     "typescript": "^5.5.3",
     "webpack": "^5.88.2",
     "webpack-cli": "^4.10.0",

packages/feel-input-component/src/FeelInput.tsx

@@ -30,7 +30,7 @@ import {
 } from "./FeelConfigs";
 
 import { FeelSyntacticSymbolNature, FeelVariables, ParsedExpression } from "@kie-tools/dmn-feel-antlr4-parser";
-import { Element } from "./themes/Element";
+import { SemanticTokensProvider } from "./semanticTokensProvider";
 
 export const EXPRESSION_PROPERTIES_SEPARATOR = ".";
 
@@ -76,23 +76,6 @@ Monaco.editor.defineTheme(MONACO_FEEL_THEME, feelTheme());
 // Don't remove this mechanism. It's necessary for Monaco to initialize correctly and display correct colors for FEEL.
 let __firstTimeInitializingMonacoToEnableColorizingCorrectly = true;
 
-function getTokenTypeIndex(symbolType: FeelSyntacticSymbolNature) {
-  switch (symbolType) {
-    default:
-    case FeelSyntacticSymbolNature.LocalVariable:
-    case FeelSyntacticSymbolNature.GlobalVariable:
-      return Element.Variable;
-    case FeelSyntacticSymbolNature.DynamicVariable:
-      return Element.DynamicVariable;
-    case FeelSyntacticSymbolNature.Unknown:
-      return Element.UnknownVariable;
-    case FeelSyntacticSymbolNature.Invocable:
-      return Element.FunctionCall;
-    case FeelSyntacticSymbolNature.Parameter:
-      return Element.FunctionParameterVariable;
-  }
-}
-
 export const FeelInput = React.forwardRef<FeelInputRef, FeelInputProps>(
   (
     {
@@ -115,6 +98,11 @@ export const FeelInput = React.forwardRef<FeelInputRef, FeelInputProps>(
 
     const [currentParsedExpression, setCurrentParsedExpression] = useState<ParsedExpression>();
 
+    const semanticTokensProvider = useMemo(
+      () => new SemanticTokensProvider(feelVariables, expressionId, setCurrentParsedExpression),
+      [expressionId, feelVariables]
+    );
+
     const getLastValidSymbolAtPosition = useCallback((currentParsedExpression: ParsedExpression, position: number) => {
       let lastValidSymbol;
       for (let i = 0; i < currentParsedExpression.feelVariables.length; i++) {
@@ -292,84 +280,7 @@ export const FeelInput = React.forwardRef<FeelInputRef, FeelInputProps>(
 
       const disposable = Monaco.languages.registerDocumentSemanticTokensProvider(
         { language: MONACO_FEEL_LANGUAGE },
-        {
-          provideDocumentSemanticTokens: function (model) {
-            const tokenTypes = new Array<number>();
-
-            if (feelVariables) {
-              const text = model.getValue();
-              const contentByLines = model.getLinesContent();
-              let startOfPreviousToken = 0;
-              let previousLine = 0;
-              let lineOffset = 0;
-              let currentLine = 0;
-              const parsedExpression = feelVariables.parser.parse(expressionId ?? "", text);
-              setCurrentParsedExpression(parsedExpression);
-
-              for (const variable of parsedExpression.feelVariables) {
-                if (variable.startLine != currentLine) {
-                  lineOffset += contentByLines[currentLine].length + 1; // +1 = the line break
-                  currentLine = variable.startLine;
-                }
-                variable.startIndex -= lineOffset;
-              }
-
-              for (const variable of parsedExpression.feelVariables) {
-                if (previousLine != variable.startLine) {
-                  startOfPreviousToken = 0;
-                }
-                if (variable.startLine === variable.endLine) {
-                  tokenTypes.push(
-                    variable.startLine - previousLine, // lineIndex = relative to the PREVIOUS line
-                    variable.startIndex - startOfPreviousToken, // columnIndex = relative to the start of the PREVIOUS token NOT to the start of the line
-                    variable.length,
-                    getTokenTypeIndex(variable.feelSymbolNature),
-                    0 // token modifier = not used so we keep it 0
-                  );
-
-                  previousLine = variable.startLine;
-                  startOfPreviousToken = variable.startIndex;
-                } else {
-                  tokenTypes.push(
-                    variable.startLine - previousLine,
-                    variable.startIndex - startOfPreviousToken,
-                    variable.length,
-                    getTokenTypeIndex(variable.feelSymbolNature),
-                    0
-                  );
-
-                  const length =
-                    variable.length - (contentByLines[variable.startLine].length - variable.startIndex + 1); // +1 = the line break
-
-                  tokenTypes.push(
-                    variable.endLine - variable.startLine,
-                    0,
-                    length,
-                    getTokenTypeIndex(variable.feelSymbolNature),
-                    0
-                  );
-
-                  startOfPreviousToken = 0;
-                  previousLine = variable.endLine;
-                }
-              }
-            }
-
-            return {
-              data: new Uint32Array(tokenTypes),
-              resultId: undefined,
-            };
-          },
-          getLegend: function (): Monaco.languages.SemanticTokensLegend {
-            return {
-              tokenTypes: Object.values(Element).filter((x) => typeof x === "string") as string[],
-              tokenModifiers: [],
-            };
-          },
-          releaseDocumentSemanticTokens: function (resultId: string | undefined): void {
-            // do nothing
-          },
-        }
+        semanticTokensProvider
       );
       return () => {
         disposable.dispose();

packages/feel-input-component/src/semanticTokensProvider.ts

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import * as Monaco from "@kie-tools-core/monaco-editor";
+import { Element } from "./themes/Element";
+import { FeelSyntacticSymbolNature, FeelVariables, ParsedExpression } from "@kie-tools/dmn-feel-antlr4-parser";
+
+export class SemanticTokensProvider implements Monaco.languages.DocumentSemanticTokensProvider {
+  constructor(
+    private feelVariables: FeelVariables | undefined,
+    private expressionId: string | undefined,
+    private setCurrentParsedExpression: (
+      value: ((prevState: ParsedExpression) => ParsedExpression) | ParsedExpression
+    ) => void
+  ) {}
+
+  onDidChange?: Monaco.IEvent<void> | undefined;
+
+  getLegend(): Monaco.languages.SemanticTokensLegend {
+    return {
+      tokenTypes: Object.values(Element).filter((x) => typeof x === "string") as string[],
+      tokenModifiers: [],
+    };
+  }
+
+  provideDocumentSemanticTokens(
+    model: Monaco.editor.ITextModel,
+    lastResultId: string | null,
+    token: Monaco.CancellationToken
+  ): Monaco.languages.ProviderResult<Monaco.languages.SemanticTokens> {
+    const tokenTypes = new Array<number>();
+
+    if (!this.feelVariables) {
+      return;
+    }
+
+    const text = model.getValue().replaceAll("\r\n", "\n");
+    const contentByLines = model.getLinesContent();
+    const parsedExpression = this.feelVariables.parser.parse(this.expressionId ?? "", text);
+
+    // This is to autocomplete, so we don't need to parse it again.
+    this.setCurrentParsedExpression(parsedExpression);
+
+    // The startIndex is set by parse relative to the ENTIRE EXPRESSION.
+    // But, here, we need a startIndex relative to LINE, because that's how Monaco works.
+    //
+    // For example, consider the expression:
+    // "a" +
+    // "b" + someVar
+    //
+    // To the parser, the index of "someVar" is 13, because it reads the expression in this format:
+    // "a" + "b" + someVar
+    //
+    // But, here, the real index of "someVar" is 7.
+    //
+    // The code bellow does this calculation fixing the startIndex solved by the parser to the
+    // startIndex we need here, relative to the LINE where the variable is, not to the full expression.
+    for (const variable of parsedExpression.feelVariables) {
+      let lineOffset = 0;
+      for (let i = 0; i < variable.startLine; i++) {
+        lineOffset += contentByLines[i].length + 1; // +1 = is the line break
+      }
+      variable.startIndex -= lineOffset;
+    }
+
+    let startOfPreviousVariable = 0;
+    let previousLine = 0;
+    for (const variable of parsedExpression.feelVariables) {
+      if (previousLine != variable.startLine) {
+        startOfPreviousVariable = 0;
+      }
+
+      // It is a variable that it is NOT split in multiple-lines
+      if (variable.startLine === variable.endLine) {
+        tokenTypes.push(
+          variable.startLine - previousLine, // lineIndex = relative to the PREVIOUS line
+          variable.startIndex - startOfPreviousVariable, // columnIndex = relative to the start of the PREVIOUS token NOT to the start of the line
+          variable.length,
+          this.getTokenTypeIndex(variable.feelSymbolNature),
+          0 // token modifier = not used so we keep it 0
+        );
+
+        previousLine = variable.startLine;
+        startOfPreviousVariable = variable.startIndex;
+      } else {
+        // It is a MULTILINE variable.
+        // We colorize the first line of the variable and then other lines.
+        tokenTypes.push(
+          variable.startLine - previousLine,
+          variable.startIndex - startOfPreviousVariable,
+          contentByLines[variable.startLine - previousLine].length - variable.startIndex,
+          this.getTokenTypeIndex(variable.feelSymbolNature),
+          0
+        );
+
+        let remainingChars =
+          variable.length - 1 - (contentByLines[variable.startLine - previousLine].length - variable.startIndex); // -1 = line break
+        const remainingLines = variable.endLine - variable.startLine;
+        let currentLine = variable.startLine + 1;
+
+        // We colorize the remaining lines here. It can be one of the following cases:
+        // 1. The entire line is part of the variable, colorize the entire line;
+        // 2. Only a few chars at the start of the currentLine is part of the variable.
+        for (let i = 0; i < remainingLines; i++) {
+          // We try to colorize everything but, if it overflows the line, it means that the variable does not end here.
+          let toColorize = remainingChars;
+          if (toColorize > contentByLines[currentLine].length) {
+            toColorize = contentByLines[currentLine].length;
+          }
+
+          tokenTypes.push(1, 0, toColorize, this.getTokenTypeIndex(variable.feelSymbolNature), 0);
+
+          remainingChars -= toColorize + 1;
+          currentLine++;
+        }
+
+        // We need to track where is the start to previous colorized variable, because it is used to calculate
+        // where we're going to paint the next variable. Monaco utilizes that as the index NOT the start of
+        // the line. So, here, we're setting it to 0 because the last painted "part of the variable"
+        // was painted at position 0 of the line.
+        startOfPreviousVariable = 0;
+        previousLine = variable.endLine;
+      }
+    }
+
+    return {
+      data: new Uint32Array(tokenTypes),
+      resultId: undefined,
+    };
+  }
+
+  releaseDocumentSemanticTokens(resultId: string | undefined): void {
+    // Do nothing
+  }
+
+  private getTokenTypeIndex(symbolType: FeelSyntacticSymbolNature) {
+    switch (symbolType) {
+      default:
+      case FeelSyntacticSymbolNature.LocalVariable:
+      case FeelSyntacticSymbolNature.GlobalVariable:
+        return Element.Variable;
+      case FeelSyntacticSymbolNature.DynamicVariable:
+        return Element.DynamicVariable;
+      case FeelSyntacticSymbolNature.Unknown:
+        return Element.UnknownVariable;
+      case FeelSyntacticSymbolNature.Invocable:
+        return Element.FunctionCall;
+      case FeelSyntacticSymbolNature.Parameter:
+        return Element.FunctionParameterVariable;
+    }
+  }
+}