[Backport 2.19-dev] Add replace command with Calcite (#4451)

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -63,8 +63,11 @@ public static String unquoteText(String text) {
     for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) {
       currentChar = text.charAt(chIndex);
       nextChar = text.charAt(chIndex + 1);
-      if (currentChar == enclosingQuote && nextChar == currentChar) {
+
+      if ((currentChar == '\\' && (nextChar == '"' || nextChar == '\\' || nextChar == '\''))
+          || (currentChar == nextChar && currentChar == enclosingQuote)) {
         chIndex++;
+        currentChar = nextChar;
       }
       textSB.append(currentChar);
     }

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -85,6 +85,7 @@
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
+import org.opensearch.sql.ast.tree.Replace;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
@@ -800,6 +801,11 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con
     return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context));
   }
 
+  @Override
+  public LogicalPlan visitReplace(Replace node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Replace");
+  }
+
   @Override
   public LogicalPlan visitJoin(Join node, AnalysisContext context) {
     throw getOnlyForCalciteException("Join");

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -73,6 +73,7 @@
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
+import org.opensearch.sql.ast.tree.Replace;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
@@ -246,6 +247,10 @@ public T visitRename(Rename node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitReplace(Replace node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitEval(Eval node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/tree/Replace.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import java.util.Set;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.jetbrains.annotations.Nullable;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+@Getter
+@Setter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class Replace extends UnresolvedPlan {
+  private final List<ReplacePair> replacePairs;
+  private final Set<Field> fieldList;
+  @Nullable private UnresolvedPlan child;
+
+  /**
+   * Constructor with multiple pattern/replacement pairs.
+   *
+   * @param replacePairs List of pattern/replacement pairs
+   * @param fieldList Set of fields to apply replacements to
+   */
+  public Replace(List<ReplacePair> replacePairs, Set<Field> fieldList) {
+    this.replacePairs = replacePairs;
+    this.fieldList = fieldList;
+  }
+
+  @Override
+  public Replace attach(UnresolvedPlan child) {
+    if (null == this.child) {
+      this.child = child;
+    } else {
+      this.child.attach(child);
+    }
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitReplace(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/ReplacePair.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.expression.Literal;
+
+/** A pair of pattern and replacement literals for the Replace command. */
+@Getter
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class ReplacePair {
+  private final Literal pattern;
+  private final Literal replacement;
+}

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -121,6 +121,8 @@
 import org.opensearch.sql.ast.tree.Regex;
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.Rename;
+import org.opensearch.sql.ast.tree.Replace;
+import org.opensearch.sql.ast.tree.ReplacePair;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
 import org.opensearch.sql.ast.tree.Search;
@@ -2414,6 +2416,51 @@ public RelNode visitValues(Values values, CalcitePlanContext context) {
     }
   }
 
+  @Override
+  public RelNode visitReplace(Replace node, CalcitePlanContext context) {
+    visitChildren(node, context);
+
+    List<String> fieldNames = context.relBuilder.peek().getRowType().getFieldNames();
+
+    // Create a set of field names to replace for quick lookup
+    Set<String> fieldsToReplace =
+        node.getFieldList().stream().map(f -> f.getField().toString()).collect(Collectors.toSet());
+
+    // Validate that all fields to replace exist by calling field() on each
+    // This leverages relBuilder.field()'s built-in validation which throws
+    // IllegalArgumentException if any field doesn't exist
+    for (String fieldToReplace : fieldsToReplace) {
+      context.relBuilder.field(fieldToReplace);
+    }
+
+    List<RexNode> projectList = new ArrayList<>();
+
+    // Project all fields, replacing specified ones in-place
+    for (String fieldName : fieldNames) {
+      if (fieldsToReplace.contains(fieldName)) {
+        // Replace this field in-place with all pattern/replacement pairs applied sequentially
+        RexNode fieldRef = context.relBuilder.field(fieldName);
+
+        // Apply all replacement pairs sequentially (nested REPLACE calls)
+        for (ReplacePair pair : node.getReplacePairs()) {
+          RexNode patternNode = rexVisitor.analyze(pair.getPattern(), context);
+          RexNode replacementNode = rexVisitor.analyze(pair.getReplacement(), context);
+          fieldRef =
+              context.relBuilder.call(
+                  SqlStdOperatorTable.REPLACE, fieldRef, patternNode, replacementNode);
+        }
+
+        projectList.add(fieldRef);
+      } else {
+        // Keep original field unchanged
+        projectList.add(context.relBuilder.field(fieldName));
+      }
+    }
+
+    context.relBuilder.project(projectList, fieldNames);
+    return context.relBuilder.peek();
+  }
+
   private void buildParseRelNode(Parse node, CalcitePlanContext context) {
     RexNode sourceField = rexVisitor.analyze(node.getSourceField(), context);
     ParseMethod parseMethod = node.getParseMethod();

docs/category.json

@@ -40,6 +40,7 @@
     "user/ppl/cmd/rare.rst",
     "user/ppl/cmd/regex.rst",
     "user/ppl/cmd/rename.rst",
+    "user/ppl/cmd/replace.rst",
     "user/ppl/cmd/rex.rst",
     "user/ppl/cmd/search.rst",
     "user/ppl/cmd/showdatasources.rst",
@@ -68,4 +69,4 @@
   "bash_settings": [
     "user/ppl/admin/settings.rst"
   ]
-}
+}

docs/user/dql/expressions.rst

@@ -46,7 +46,7 @@ Here is an example for different type of literals::
     +---------+---------+---------+---------+--------+---------+---------+-----------+----------+
     | "Hello" | 'Hello' | "It""s" | 'It''s' | "It's" | '"Its"' | 'It\'s' | 'It\\\'s' | "\I\t\s" |
     |---------+---------+---------+---------+--------+---------+---------+-----------+----------|
-    | Hello   | Hello   | It"s    | It's    | It's   | "Its"   | It\'s   | It\\\'s   | \I\t\s   |
+    | Hello   | Hello   | It"s    | It's    | It's   | "Its"   | It's    | It\'s     | \I\t\s   |
     +---------+---------+---------+---------+--------+---------+---------+-----------+----------+
 
 

docs/user/ppl/cmd/replace.rst

@@ -0,0 +1,127 @@
+=============
+replace
+=============
+
+.. rubric:: Table of contents
+
+.. contents::
+ :local:
+ :depth: 2
+
+
+Description
+============
+Using ``replace`` command to replace text in one or more fields in the search result.
+
+Note: This command is only available when Calcite engine is enabled.
+
+
+Syntax
+============
+replace '<pattern>' WITH '<replacement>' [, '<pattern>' WITH '<replacement>']... IN <field-name>[, <field-name>]...
+
+
+Parameters
+==========
+* **pattern**: mandatory. The text pattern you want to replace. Currently supports only plain text literals (no wildcards or regular expressions).
+* **replacement**: mandatory. The text you want to replace with.
+* **field-name**: mandatory. One or more field names where the replacement should occur.
+
+
+Examples
+========
+
+Example 1: Replace text in one field
+------------------------------------
+
+The example shows replacing text in one field.
+
+PPL query::
+
+ os> source=accounts | replace "IL" WITH "Illinois" IN state | fields state;
+ fetched rows / total rows = 4/4
+ +----------+
+ | state    |
+ |----------|
+ | Illinois |
+ | TN       |
+ | VA       |
+ | MD       |
+ +----------+
+
+
+Example 2: Replace text in multiple fields
+------------------------------------
+
+The example shows replacing text in multiple fields.
+
+PPL query::
+
+ os> source=accounts | replace "IL" WITH "Illinois" IN state, address | fields state, address;
+ fetched rows / total rows = 4/4
+ +----------+----------------------+
+ | state    | address              |
+ |----------+----------------------|
+ | Illinois | 880 Holmes Lane      |
+ | TN       | 671 Bristol Street   |
+ | VA       | 789 Madison Street   |
+ | MD       | 467 Hutchinson Court |
+ +----------+----------------------+
+
+
+Example 3: Replace with other commands in a pipeline
+------------------------------------
+
+The example shows using replace with other commands in a query pipeline.
+
+PPL query::
+
+ os> source=accounts | replace "IL" WITH "Illinois" IN state | where age > 30 | fields state, age;
+ fetched rows / total rows = 3/3
+ +----------+-----+
+ | state    | age |
+ |----------+-----|
+ | Illinois | 32  |
+ | TN       | 36  |
+ | MD       | 33  |
+ +----------+-----+
+
+Example 4: Replace with multiple pattern/replacement pairs
+------------------------------------
+
+The example shows using multiple pattern/replacement pairs in a single replace command. The replacements are applied sequentially.
+
+PPL query::
+
+ os> source=accounts | replace "IL" WITH "Illinois", "TN" WITH "Tennessee" IN state | fields state;
+ fetched rows / total rows = 4/4
+ +-----------+
+ | state     |
+ |-----------|
+ | Illinois  |
+ | Tennessee |
+ | VA        |
+ | MD        |
+ +-----------+
+
+Example 5: Pattern matching with LIKE and replace
+------------------------------------
+
+Since replace command only supports plain string literals, you can use LIKE command with replace for pattern matching needs.
+
+PPL query::
+
+ os> source=accounts | where LIKE(address, '%Holmes%') | replace "Holmes" WITH "HOLMES" IN address | fields address, state, gender, age, city;
+ fetched rows / total rows = 1/1
+ +-----------------+-------+--------+-----+--------+
+ | address         | state | gender | age | city   |
+ |-----------------+-------+--------+-----+--------|
+ | 880 HOLMES Lane | IL    | M      | 32  | Brogan |
+ +-----------------+-------+--------+-----+--------+
+
+
+Limitations
+===========
+* Only supports plain text literals for pattern matching. Wildcards and regular expressions are not supported.
+* Pattern and replacement values must be string literals.
+* The replace command modifies the specified fields in-place.

docs/user/ppl/functions/string.rst

@@ -215,6 +215,14 @@ Argument type: STRING, STRING (regex pattern), STRING (replacement)
 
 Return type: STRING
 
+**Important - Regex Special Characters**: The pattern is interpreted as a regular expression. Characters like ``.``, ``*``, ``+``, ``[``, ``]``, ``(``, ``)``, ``{``, ``}``, ``^``, ``$``, ``|``, ``?``, and ``\`` have special meaning in regex. To match them literally, escape with backslashes:
+
+* To match ``example.com``: use ``'example\\.com'`` (escape the dots)
+* To match ``value*``: use ``'value\\*'`` (escape the asterisk)
+* To match ``price+tax``: use ``'price\\+tax'`` (escape the plus)
+
+For strings with many special characters, use ``\\Q...\\E`` to quote the entire literal string (e.g., ``'\\Qhttps://example.com/path?id=123\\E'`` matches that exact URL).
+
 Literal String Replacement Examples::
 
     os> source=people | eval `REPLACE('helloworld', 'world', 'universe')` = REPLACE('helloworld', 'world', 'universe'), `REPLACE('helloworld', 'invalid', 'universe')` = REPLACE('helloworld', 'invalid', 'universe') | fields `REPLACE('helloworld', 'world', 'universe')`, `REPLACE('helloworld', 'invalid', 'universe')`
@@ -225,6 +233,16 @@ Literal String Replacement Examples::
     | hellouniverse                              | helloworld                                   |
     +--------------------------------------------+----------------------------------------------+
 
+Escaping Special Characters Examples::
+
+    os> source=people | eval `Replace domain` = REPLACE('api.example.com', 'example\\.com', 'newsite.org'), `Replace with quote` = REPLACE('https://api.example.com/v1', '\\Qhttps://api.example.com\\E', 'http://localhost:8080') | fields `Replace domain`, `Replace with quote`
+    fetched rows / total rows = 1/1
+    +-----------------+--------------------------+
+    | Replace domain  | Replace with quote       |
+    |-----------------+--------------------------|
+    | api.newsite.org | http://localhost:8080/v1 |
+    +-----------------+--------------------------+
+
 Regex Pattern Examples::
 
     os> source=people | eval `Remove digits` = REPLACE('test123', '\d+', ''), `Collapse spaces` = REPLACE('hello  world', ' +', ' '), `Remove special` = REPLACE('hello@world!', '[^a-zA-Z]', '') | fields `Remove digits`, `Collapse spaces`, `Remove special`

docs/user/ppl/index.rst

@@ -122,6 +122,8 @@ The query start with search command and then flowing a set of command delimited
 
   - `trendline command <cmd/trendline.rst>`_
 
+  - `replace command <cmd/replace.rst>`_
+
   - `where command <cmd/where.rst>`_
 
 * **Functions**

integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java

@@ -89,6 +89,7 @@
   CalciteRegexCommandIT.class,
   CalciteRexCommandIT.class,
   CalciteRenameCommandIT.class,
+  CalciteReplaceCommandIT.class,
   CalciteResourceMonitorIT.class,
   CalciteSearchCommandIT.class,
   CalciteSettingsIT.class,