Dev dbg mitchell branch

.github/workflows/sql-test-and-build-workflow.yml

@@ -22,9 +22,6 @@ jobs:
     - name: Build with Gradle
       run: ./gradlew build assemble
 
-    - name: Run backward compatibility tests
-      run: ./scripts/bwctest.sh
-
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -22,45 +22,45 @@ jobs:
         with:
           repository: opensearch-project/Opensearch-Dashboards
           ref: ${{ env.OPENSEARCH_VERSION }}
-          path: sql/OpenSearch-Dashboards
+          path: OpenSearch-Dashboards
 
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version-file: "./OpenSearch-Dashboards/.nvmrc"
+          node-version-file: "../OpenSearch-Dashboards/.nvmrc"
           registry-url: 'https://registry.npmjs.org'
 
       - name: Move Workbench to Plugins Dir
         run: |
-          mv workbench OpenSearch-Dashboards/plugins
+          mv workbench ../OpenSearch-Dashboards/plugins
 
       - name: OpenSearch Dashboards Plugin Bootstrap
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 60
           max_attempts: 3
-          command: cd OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap
+          command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap
 
       - name: Test
         run: |
-          cd OpenSearch-Dashboards/plugins/workbench
+          cd ../OpenSearch-Dashboards/plugins/workbench
           yarn test:jest --coverage
 
       - name: Upload coverage
         uses: codecov/codecov-action@v1
         with:
           flags: query-workbench
-          directory: ./OpenSearch-Dashboards/plugins/workbench
+          directory: ../OpenSearch-Dashboards/plugins/workbench
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Build Artifact
         run: |
-          cd OpenSearch-Dashboards/plugins/workbench
+          cd ../OpenSearch-Dashboards/plugins/workbench
           yarn build
           mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v1
         with:
           name: workbench
-          path: OpenSearch-Dashboards/plugins/workbench/build
+          path: ../OpenSearch-Dashboards/plugins/workbench/build

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -27,16 +27,54 @@ public static String unquote(String text, String mark) {
 
   /**
    * Unquote Identifier which has " or ' or ` as mark.
+   * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote
+   * acts as an escape
+   * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced
+   * with double.
    * @param text string
    * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been
    *     removed
    */
   public static String unquoteText(String text) {
-    if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) {
-      return text.substring(1, text.length() - 1);
+
+    if (text.length() < 2) {
+      return text;
+    }
+
+    char enclosingQuote;
+    char firstChar = text.charAt(0);
+    char lastChar = text.charAt(text.length() - 1);
+
+    if (firstChar == lastChar
+        && (firstChar == '\''
+        || firstChar == '"'
+        || firstChar == '`')) {
+      enclosingQuote = firstChar;
     } else {
       return text;
     }
+
+    if (enclosingQuote == '`') {
+      return text.substring(1, text.length() - 1);
+    }
+
+    char currentChar;
+    char nextChar;
+
+    StringBuilder textSB = new StringBuilder();
+
+    // Ignores first and last character as they are the quotes that should be removed
+    for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) {
+      currentChar = text.charAt(chIndex);
+      nextChar = text.charAt(chIndex + 1);
+      if (currentChar == enclosingQuote
+          && nextChar == currentChar) {
+        chIndex++;
+      }
+      textSB.append(currentChar);
+    }
+
+    return textSB.toString();
   }
 
   /**

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -292,6 +292,12 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) {
       child = windowAnalyzer.analyze(expr, context);
     }
 
+    for (UnresolvedExpression expr : node.getProjectList()) {
+      HighlightAnalyzer highlightAnalyzer = new HighlightAnalyzer(expressionAnalyzer, child);
+      child = highlightAnalyzer.analyze(expr, context);
+
+    }
+
     List<NamedExpression> namedExpressions =
         selectExpressionAnalyzer.analyze(node.getProjectList(), context,
             new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child));

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -11,9 +11,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import lombok.Getter;
@@ -29,6 +27,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Literal;
@@ -44,12 +43,12 @@
 import org.opensearch.sql.ast.expression.WindowFunction;
 import org.opensearch.sql.ast.expression.Xor;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
-import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
 import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.expression.HighlightExpression;
 import org.opensearch.sql.expression.LiteralExpression;
 import org.opensearch.sql.expression.NamedArgumentExpression;
 import org.opensearch.sql.expression.NamedExpression;
@@ -191,6 +190,12 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte
     return expr;
   }
 
+  @Override
+  public Expression visitHighlight(HighlightFunction node, AnalysisContext context) {
+    Expression expr = node.getHighlightField().accept(this, context);
+    return new HighlightExpression(expr);
+  }
+
   @Override
   public Expression visitIn(In node, AnalysisContext context) {
     return visitIn(node.getField(), node.getValueList(), context);

core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.analysis;
+
+import lombok.RequiredArgsConstructor;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Alias;
+import org.opensearch.sql.ast.expression.HighlightFunction;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.planner.logical.LogicalHighlight;
+import org.opensearch.sql.planner.logical.LogicalPlan;
+
+/**
+ * Analyze the highlight in the {@link AnalysisContext} to construct the {@link
+ * LogicalPlan}.
+ */
+@RequiredArgsConstructor
+public class HighlightAnalyzer extends AbstractNodeVisitor<LogicalPlan, AnalysisContext> {
+  private final ExpressionAnalyzer expressionAnalyzer;
+  private final LogicalPlan child;
+
+  public LogicalPlan analyze(UnresolvedExpression projectItem, AnalysisContext context) {
+    LogicalPlan highlight = projectItem.accept(this, context);
+    return (highlight == null) ? child : highlight;
+  }
+
+  @Override
+  public LogicalPlan visitAlias(Alias node, AnalysisContext context) {
+    if (!(node.getDelegated() instanceof HighlightFunction)) {
+      return null;
+    }
+
+    HighlightFunction unresolved = (HighlightFunction) node.getDelegated();
+    Expression field = expressionAnalyzer.analyze(unresolved.getHighlightField(), context);
+    return new LogicalHighlight(child, field);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -18,6 +18,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Let;
@@ -254,4 +255,8 @@ public T visitKmeans(Kmeans node, C context) {
   public T visitAD(AD node, C context) {
     return visitChildren(node, context);
   }
+
+  public T visitHighlight(HighlightFunction node, C context) {
+    return visitChildren(node, context);
+  }
 }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -22,6 +22,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Let;
@@ -261,6 +262,10 @@ public When when(UnresolvedExpression condition, UnresolvedExpression result) {
     return new When(condition, result);
   }
 
+  public UnresolvedExpression highlight(UnresolvedExpression fieldName) {
+    return new HighlightFunction(fieldName);
+  }
+
   public UnresolvedExpression window(UnresolvedExpression function,
                                      List<UnresolvedExpression> partitionByList,
                                      List<Pair<SortOption, UnresolvedExpression>> sortList) {

core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+
+/**
+ * Expression node of Highlight function.
+ */
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Getter
+@ToString
+public class HighlightFunction extends UnresolvedExpression {
+  private final UnresolvedExpression highlightField;
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitHighlight(this, context);
+  }
+
+  @Override
+  public List<UnresolvedExpression> getChild() {
+    return List.of(highlightField);
+  }
+}

core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java

@@ -55,6 +55,10 @@ public T visitNamed(NamedExpression node, C context) {
     return node.getDelegated().accept(this, context);
   }
 
+  public T visitHighlight(HighlightExpression node, C context) {
+    return visitNode(node, context);
+  }
+
   public T visitReference(ReferenceExpression node, C context) {
     return visitNode(node, context);
   }

core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.expression;
+
+import java.util.List;
+import lombok.Getter;
+import org.opensearch.sql.common.utils.StringUtils;
+import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.type.ExprCoreType;
+import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.expression.env.Environment;
+import org.opensearch.sql.expression.function.BuiltinFunctionName;
+
+/**
+ * Highlight Expression.
+ */
+@Getter
+public class HighlightExpression extends FunctionExpression {
+  private final Expression highlightField;
+
+  /**
+   * HighlightExpression Constructor.
+   * @param highlightField : Highlight field for expression.
+   */
+  public HighlightExpression(Expression highlightField) {
+    super(BuiltinFunctionName.HIGHLIGHT.getName(), List.of(highlightField));
+    this.highlightField = highlightField;
+  }
+
+  /**
+   * Return collection value matching highlight field.
+   * @param valueEnv : Dataset to parse value from.
+   * @return : collection value of highlight fields.
+   */
+  @Override
+  public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
+    String refName = "_highlight" + "." + StringUtils.unquoteText(getHighlightField().toString());
+    return valueEnv.resolve(DSL.ref(refName, ExprCoreType.STRING));
+  }
+
+  /**
+   * Get type for HighlightExpression.
+   * @return : String type.
+   */
+  @Override
+  public ExprType type() {
+    return ExprCoreType.ARRAY;
+  }
+
+  @Override
+  public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
+    return visitor.visitHighlight(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -192,6 +192,7 @@ public enum BuiltinFunctionName {
   MATCHPHRASE(FunctionName.of("matchphrase")),
   QUERY_STRING(FunctionName.of("query_string")),
   MATCH_BOOL_PREFIX(FunctionName.of("match_bool_prefix")),
+  HIGHLIGHT(FunctionName.of("highlight")),
   MATCH_PHRASE_PREFIX(FunctionName.of("match_phrase_prefix")),
   /**
    * Legacy Relevance Function.