Merge main to 2.x branch

.github/workflows/draft-release-notes-workflow.yml

@@ -16,6 +16,6 @@ jobs:
         with:
           config-name: draft-release-notes-config.yml
           tag: (None)
-          version: 2.1.0.0
+          version: 2.2.0.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sql-odbc-release-workflow.yml

@@ -12,7 +12,7 @@ env:
   ODBC_BUILD_PATH: "./build/odbc/build"
   AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
   PLUGIN_NAME: opensearch-sql-odbc
-  OD_VERSION: 2.1.0.0
+  OD_VERSION: 2.2.0.0
 
 jobs:
   build-mac:

.github/workflows/sql-workbench-release-workflow.yml

@@ -8,7 +8,7 @@ on:
 env: 
   PLUGIN_NAME: query-workbench-dashboards
   OPENSEARCH_VERSION: 'main'
-  OPENSEARCH_PLUGIN_VERSION: 2.1.0.0
+  OPENSEARCH_PLUGIN_VERSION: 2.2.0.0
 
 jobs:
 

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -5,7 +5,7 @@ on: [pull_request, push]
 env: 
   PLUGIN_NAME: query-workbench-dashboards
   OPENSEARCH_VERSION: 'main'
-  OPENSEARCH_PLUGIN_VERSION: 2.1.0.0
+  OPENSEARCH_PLUGIN_VERSION: 2.2.0.0
 
 jobs:
 
@@ -22,45 +22,45 @@ jobs:
         with:
           repository: opensearch-project/Opensearch-Dashboards
           ref: ${{ env.OPENSEARCH_VERSION }}
-          path: sql/OpenSearch-Dashboards
+          path: OpenSearch-Dashboards
 
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version-file: "./OpenSearch-Dashboards/.nvmrc"
+          node-version-file: "../OpenSearch-Dashboards/.nvmrc"
           registry-url: 'https://registry.npmjs.org'
 
       - name: Move Workbench to Plugins Dir
         run: |
-          mv workbench OpenSearch-Dashboards/plugins
+          mv workbench ../OpenSearch-Dashboards/plugins
 
       - name: OpenSearch Dashboards Plugin Bootstrap
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 60
           max_attempts: 3
-          command: cd OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap
+          command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap
 
       - name: Test
         run: |
-          cd OpenSearch-Dashboards/plugins/workbench
+          cd ../OpenSearch-Dashboards/plugins/workbench
           yarn test:jest --coverage
 
       - name: Upload coverage
         uses: codecov/codecov-action@v1
         with:
           flags: query-workbench
-          directory: ./OpenSearch-Dashboards/plugins/workbench
+          directory: ../OpenSearch-Dashboards/plugins/workbench
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Build Artifact
         run: |
-          cd OpenSearch-Dashboards/plugins/workbench
+          cd ../OpenSearch-Dashboards/plugins/workbench
           yarn build
           mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip
 
       - name: Upload Artifact
         uses: actions/upload-artifact@v1
         with:
           name: workbench
-          path: OpenSearch-Dashboards/plugins/workbench/build
+          path: ../OpenSearch-Dashboards/plugins/workbench/build

build.gradle

@@ -6,10 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.1.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT")
         spring_version = "5.3.22"
         jackson_version = "2.13.3"
-        jackson_databind_version = "2.13.3"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -137,11 +136,6 @@ subprojects {
                 "org.checkstyle.google.suppressionfilter.config": rootProject.file("config/checkstyle/suppressions.xml")]
         ignoreFailures = false
     }
-
-    configurations.all {
-        resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jackson_version}"
-        resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson_version}"
-    }
 }
 checkstyle {
     configFile file("config/checkstyle/checkstyle.xml")

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -27,16 +27,54 @@ public static String unquote(String text, String mark) {
 
   /**
    * Unquote Identifier which has " or ' or ` as mark.
+   * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote
+   * acts as an escape
+   * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced
+   * with double.
    * @param text string
    * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been
    *     removed
    */
   public static String unquoteText(String text) {
-    if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) {
-      return text.substring(1, text.length() - 1);
+
+    if (text.length() < 2) {
+      return text;
+    }
+
+    char enclosingQuote;
+    char firstChar = text.charAt(0);
+    char lastChar = text.charAt(text.length() - 1);
+
+    if (firstChar == lastChar
+        && (firstChar == '\''
+        || firstChar == '"'
+        || firstChar == '`')) {
+      enclosingQuote = firstChar;
     } else {
       return text;
     }
+
+    if (enclosingQuote == '`') {
+      return text.substring(1, text.length() - 1);
+    }
+
+    char currentChar;
+    char nextChar;
+
+    StringBuilder textSB = new StringBuilder();
+
+    // Ignores first and last character as they are the quotes that should be removed
+    for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) {
+      currentChar = text.charAt(chIndex);
+      nextChar = text.charAt(chIndex + 1);
+      if (currentChar == enclosingQuote
+          && nextChar == currentChar) {
+        chIndex++;
+      }
+      textSB.append(currentChar);
+    }
+
+    return textSB.toString();
   }
 
   /**

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -292,6 +292,12 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) {
       child = windowAnalyzer.analyze(expr, context);
     }
 
+    for (UnresolvedExpression expr : node.getProjectList()) {
+      HighlightAnalyzer highlightAnalyzer = new HighlightAnalyzer(expressionAnalyzer, child);
+      child = highlightAnalyzer.analyze(expr, context);
+
+    }
+
     List<NamedExpression> namedExpressions =
         selectExpressionAnalyzer.analyze(node.getProjectList(), context,
             new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child));

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -11,9 +11,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import lombok.Getter;
@@ -29,6 +27,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Literal;
@@ -44,12 +43,12 @@
 import org.opensearch.sql.ast.expression.WindowFunction;
 import org.opensearch.sql.ast.expression.Xor;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
-import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprType;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
 import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.expression.HighlightExpression;
 import org.opensearch.sql.expression.LiteralExpression;
 import org.opensearch.sql.expression.NamedArgumentExpression;
 import org.opensearch.sql.expression.NamedExpression;
@@ -191,6 +190,12 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte
     return expr;
   }
 
+  @Override
+  public Expression visitHighlight(HighlightFunction node, AnalysisContext context) {
+    Expression expr = node.getHighlightField().accept(this, context);
+    return new HighlightExpression(expr);
+  }
+
   @Override
   public Expression visitIn(In node, AnalysisContext context) {
     return visitIn(node.getField(), node.getValueList(), context);

core/src/main/java/org/opensearch/sql/analysis/HighlightAnalyzer.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.analysis;
+
+import lombok.RequiredArgsConstructor;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Alias;
+import org.opensearch.sql.ast.expression.HighlightFunction;
+import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.expression.Expression;
+import org.opensearch.sql.planner.logical.LogicalHighlight;
+import org.opensearch.sql.planner.logical.LogicalPlan;
+
+/**
+ * Analyze the highlight in the {@link AnalysisContext} to construct the {@link
+ * LogicalPlan}.
+ */
+@RequiredArgsConstructor
+public class HighlightAnalyzer extends AbstractNodeVisitor<LogicalPlan, AnalysisContext> {
+  private final ExpressionAnalyzer expressionAnalyzer;
+  private final LogicalPlan child;
+
+  public LogicalPlan analyze(UnresolvedExpression projectItem, AnalysisContext context) {
+    LogicalPlan highlight = projectItem.accept(this, context);
+    return (highlight == null) ? child : highlight;
+  }
+
+  @Override
+  public LogicalPlan visitAlias(Alias node, AnalysisContext context) {
+    if (!(node.getDelegated() instanceof HighlightFunction)) {
+      return null;
+    }
+
+    HighlightFunction unresolved = (HighlightFunction) node.getDelegated();
+    Expression field = expressionAnalyzer.analyze(unresolved.getHighlightField(), context);
+    return new LogicalHighlight(child, field);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -18,6 +18,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Let;
@@ -254,4 +255,8 @@ public T visitKmeans(Kmeans node, C context) {
   public T visitAD(AD node, C context) {
     return visitChildren(node, context);
   }
+
+  public T visitHighlight(HighlightFunction node, C context) {
+    return visitChildren(node, context);
+  }
 }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -22,6 +22,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Let;
@@ -261,6 +262,10 @@ public When when(UnresolvedExpression condition, UnresolvedExpression result) {
     return new When(condition, result);
   }
 
+  public UnresolvedExpression highlight(UnresolvedExpression fieldName) {
+    return new HighlightFunction(fieldName);
+  }
+
   public UnresolvedExpression window(UnresolvedExpression function,
                                      List<UnresolvedExpression> partitionByList,
                                      List<Pair<SortOption, UnresolvedExpression>> sortList) {

core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+
+/**
+ * Expression node of Highlight function.
+ */
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Getter
+@ToString
+public class HighlightFunction extends UnresolvedExpression {
+  private final UnresolvedExpression highlightField;
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitHighlight(this, context);
+  }
+
+  @Override
+  public List<UnresolvedExpression> getChild() {
+    return List.of(highlightField);
+  }
+}

core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java

@@ -55,6 +55,10 @@ public T visitNamed(NamedExpression node, C context) {
     return node.getDelegated().accept(this, context);
   }
 
+  public T visitHighlight(HighlightExpression node, C context) {
+    return visitNode(node, context);
+  }
+
   public T visitReference(ReferenceExpression node, C context) {
     return visitNode(node, context);
   }