Support spath with dynamic fields

common/src/main/java/org/opensearch/sql/common/utils/DebugUtils.java

@@ -8,8 +8,6 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Collectors;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 /**
  * Utility class for debugging operations. This class is only for debugging purpose, and not
@@ -18,7 +16,6 @@
 public class DebugUtils {
   // Update this to true while you are debugging. (Safe guard to avoid usage in production code. )
   private static final boolean IS_DEBUG = false;
-  private static final Logger logger = LogManager.getLogger(DebugUtils.class);
 
   public static <T> T debug(T obj, String message) {
     verifyDebug();
@@ -39,7 +36,7 @@ private static void verifyDebug() {
   }
 
   private static void print(String format, Object... args) {
-    logger.info(String.format(format, args));
+    System.out.println(String.format(format, args));
   }
 
   private static String getCalledFrom(int pos) {

core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionResult.java

@@ -28,17 +28,17 @@ public class FieldResolutionResult {
   @NonNull private final Set<String> regularFields;
   @NonNull private final Wildcard wildcard;
 
-  public FieldResolutionResult(Set<String> regularFields) {
+  public FieldResolutionResult(Collection<String> regularFields) {
     this.regularFields = new HashSet<>(regularFields);
     this.wildcard = NULL_WILDCARD;
   }
 
-  public FieldResolutionResult(Set<String> regularFields, Wildcard wildcard) {
+  public FieldResolutionResult(Collection<String> regularFields, Wildcard wildcard) {
     this.regularFields = new HashSet<>(regularFields);
     this.wildcard = wildcard;
   }
 
-  public FieldResolutionResult(Set<String> regularFields, String wildcard) {
+  public FieldResolutionResult(Collection<String> regularFields, String wildcard) {
     this.regularFields = new HashSet<>(regularFields);
     this.wildcard = getWildcard(wildcard);
   }
@@ -53,12 +53,12 @@ private static Wildcard getWildcard(String wildcard) {
     }
   }
 
-  public FieldResolutionResult(Set<String> regularFields, Set<String> wildcards) {
+  public FieldResolutionResult(Collection<String> regularFields, Collection<String> wildcards) {
     this.regularFields = new HashSet<>(regularFields);
     this.wildcard = createOrWildcard(wildcards);
   }
 
-  private static Wildcard createOrWildcard(Set<String> patterns) {
+  private static Wildcard createOrWildcard(Collection<String> patterns) {
     if (patterns == null || patterns.isEmpty()) {
       return NULL_WILDCARD;
     }
@@ -70,38 +70,50 @@ private static Wildcard createOrWildcard(Set<String> patterns) {
     return new OrWildcard(wildcards);
   }
 
+  /** Returns unmodifiable view of regular fields. */
   public Set<String> getRegularFieldsUnmodifiable() {
     return Collections.unmodifiableSet(regularFields);
   }
 
+  /** Checks if result contains any wildcard patterns. */
   public boolean hasWildcards() {
     return wildcard != NULL_WILDCARD;
   }
 
+  /** Checks if result contains partial wildcard patterns (not '*'). */
+  public boolean hasPartialWildcards() {
+    return wildcard != NULL_WILDCARD && wildcard != ANY_WILDCARD;
+  }
+
+  /** Checks if result contains regular fields. */
   public boolean hasRegularFields() {
     return !regularFields.isEmpty();
   }
 
+  /** Creates new result excluding specified fields. */
   public FieldResolutionResult exclude(Collection<String> fields) {
     Set<String> combinedFields = new HashSet<>(this.regularFields);
     combinedFields.removeAll(fields);
     return new FieldResolutionResult(combinedFields, this.wildcard);
   }
 
-  public FieldResolutionResult or(Set<String> fields) {
+  /** Creates new result combining this result with additional fields (union). */
+  public FieldResolutionResult or(Collection<String> fields) {
     Set<String> combinedFields = new HashSet<>(this.regularFields);
     combinedFields.addAll(fields);
     return new FieldResolutionResult(combinedFields, this.wildcard);
   }
 
-  private Set<String> and(Set<String> fields) {
+  private Set<String> and(Collection<String> fields) {
     return fields.stream()
         .filter(field -> this.getRegularFields().contains(field) || this.wildcard.matches(field))
         .collect(Collectors.toSet());
   }
 
+  /** Creates new result intersecting this result with another (intersection). */
   public FieldResolutionResult and(FieldResolutionResult other) {
-    Set<String> combinedFields = this.and(other.regularFields);
+    Set<String> combinedFields = new HashSet<>();
+    combinedFields.addAll(this.and(other.regularFields));
     combinedFields.addAll(other.and(this.regularFields));
 
     Wildcard combinedWildcard = this.wildcard.and(other.wildcard);
@@ -111,6 +123,7 @@ public FieldResolutionResult and(FieldResolutionResult other) {
 
   /** Interface for wildcard pattern matching. */
   public interface Wildcard {
+    /** Checks if field name matches wildcard pattern. */
     boolean matches(String fieldName);
 
     default Wildcard and(Wildcard other) {

core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java

@@ -75,6 +75,8 @@
  */
 public class FieldResolutionVisitor extends AbstractNodeVisitor<Node, FieldResolutionContext> {
 
+  private static final String ALL_FIELDS = "*";
+
   /**
    * Analyzes PPL query plan to determine required fields at each node.
    *
@@ -110,10 +112,10 @@ private void acceptAndVerifyNodeVisited(Node node, FieldResolutionContext contex
 
   @Override
   public Node visitProject(Project node, FieldResolutionContext context) {
-    boolean isSelectAll =
-        node.getProjectList().stream().anyMatch(expr -> expr instanceof AllFields);
+    boolean isSingleSelectAll =
+        node.getProjectList().size() == 1 && node.getProjectList().get(0) instanceof AllFields;
 
-    if (isSelectAll) {
+    if (isSingleSelectAll) {
       visitChildren(node, context);
     } else {
       Set<String> projectFields = new HashSet<>();
@@ -179,15 +181,14 @@ public Node visitSpath(SPath node, FieldResolutionContext context) {
       return visitEval(node.rewriteAsEval(), context);
     } else {
       // set requirements for spath command;
-      context.setResult(node, context.getCurrentRequirements());
       FieldResolutionResult requirements = context.getCurrentRequirements();
-      if (requirements.hasWildcards()) {
+      context.setResult(node, requirements);
+      if (requirements.hasPartialWildcards()) {
         throw new IllegalArgumentException(
-            "Spath command cannot extract arbitrary fields. Please project fields explicitly by"
-                + " fields command without wildcard or stats command.");
+            "Spath command cannot be used with partial wildcard such as `prefix*`.");
       }
 
-      context.pushRequirements(context.getCurrentRequirements().or(Set.of(node.getInField())));
+      context.pushRequirements(requirements.or(Set.of(node.getInField())));
       visitChildren(node, context);
       context.popRequirements();
       return node;
@@ -237,6 +238,8 @@ private Set<String> extractFieldsFromExpression(UnresolvedExpression expr) {
 
     if (expr instanceof Field field) {
       fields.add(field.getField().toString());
+    } else if (expr instanceof AllFields) {
+      fields.add(ALL_FIELDS);
     } else if (expr instanceof QualifiedName name) {
       fields.add(name.toString());
     } else if (expr instanceof Alias alias) {
@@ -490,43 +493,56 @@ public Node visitStreamWindow(StreamWindow node, FieldResolutionContext context)
 
   @Override
   public Node visitFillNull(FillNull node, FieldResolutionContext context) {
+    if (node.isAgainstAllFields()) {
+      throw new IllegalArgumentException("Fields need to be specified with fillnull command");
+    }
+    Set<String> fields = new HashSet<>();
+    node.getFields().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));
+
+    context.pushRequirements(context.getCurrentRequirements().or(fields));
     visitChildren(node, context);
+    context.popRequirements();
     return node;
   }
 
   @Override
   public Node visitAppendCol(AppendCol node, FieldResolutionContext context) {
-    visitChildren(node, context);
-    return node;
+    throw new IllegalArgumentException(
+        "AppendCol command cannot be used together with spath command");
   }
 
   @Override
   public Node visitAppend(Append node, FieldResolutionContext context) {
+    // dispatch requirements to subsearch and main
+    acceptAndVerifyNodeVisited(node.getSubSearch(), context);
     visitChildren(node, context);
     return node;
   }
 
   @Override
   public Node visitMultisearch(Multisearch node, FieldResolutionContext context) {
-    visitChildren(node, context);
-    return node;
+    throw new IllegalArgumentException(
+        "Multisearch command cannot be used together with spath command");
   }
 
   @Override
   public Node visitLookup(Lookup node, FieldResolutionContext context) {
-    visitChildren(node, context);
-    return node;
+    throw new IllegalArgumentException("Lookup command cannot be used together with spath command");
   }
 
   @Override
   public Node visitValues(Values node, FieldResolutionContext context) {
-    visitChildren(node, context);
-    return node;
+    throw new IllegalArgumentException("Values command cannot be used together with spath command");
   }
 
   @Override
   public Node visitReplace(Replace node, FieldResolutionContext context) {
+    Set<String> fields = new HashSet<>();
+    node.getFieldList().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));
+
+    context.pushRequirements(context.getCurrentRequirements().or(fields));
     visitChildren(node, context);
+    context.popRequirements();
     return node;
   }
 
@@ -619,6 +635,10 @@ private Set<String> extractFieldsFromAggregation(UnresolvedExpression expr) {
         }
       }
     }
-    return fields;
+    return excludeAllFieldsWildcard(fields);
+  }
+
+  private Set<String> excludeAllFieldsWildcard(Set<String> fields) {
+    return fields.stream().filter(f -> !f.equals(ALL_FIELDS)).collect(Collectors.toSet());
   }
 }

core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java

@@ -63,6 +63,10 @@ public List<Field> getFields() {
     return getReplacementPairs().stream().map(Pair::getLeft).toList();
   }
 
+  public boolean isAgainstAllFields() {
+    return !replacementForAll.isEmpty() && getReplacementPairs().isEmpty();
+  }
+
   @Override
   public FillNull attach(UnresolvedPlan child) {
     this.child = child;

core/src/main/java/org/opensearch/sql/ast/tree/Join.java

@@ -17,6 +17,7 @@
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.expression.Argument;
 import org.opensearch.sql.ast.expression.Field;
+import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
 @ToString
@@ -87,6 +88,14 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
     return nodeVisitor.visitJoin(this, context);
   }
 
+  /**
+   * @return `overwrite` option value in argumentMap
+   */
+  public boolean isOverwrite() {
+    return getArgumentMap().get("overwrite") == null // 'overwrite' default value is true
+        || getArgumentMap().get("overwrite").equals(Literal.TRUE);
+  }
+
   public enum JoinType {
     INNER,
     LEFT,