ESQL: Support intra-row field references in ROW command

docs/changelog/140217.yaml

@@ -0,0 +1,6 @@
+pr: 140217
+summary: "ESQL: Support intra-row field references in ROW command"
+area: ES|QL
+type: feature
+issues:
+  - 140119

x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec

@@ -353,3 +353,100 @@ row x = cidr_match(to_ip("127.0.0.1"), "127.0.1.0/16"), y = cidr_match(to_ip("12
 x:boolean | y:boolean  |  ip:ip     | z:boolean
 true      |false       |127.0.0.1   |true        
 ;
+
+// Test ROW field resolution - basic case
+rowFieldResolutionBasic
+ROW x = 4, y = 2, z = x + y;
+
+x:integer | y:integer | z:integer
+4         | 2         | 6
+;
+
+// Test ROW field resolution - multiple references
+rowFieldResolutionMultipleRefs
+ROW a = 10, b = a * 2, c = a + b, d = b - a;
+
+a:integer | b:integer | c:integer | d:integer
+10        | 20        | 30        | 10
+;
+
+// Test ROW field resolution - complex expressions
+rowFieldResolutionComplexExpr
+ROW x = 5, y = 3, z = x * y + 10, w = z / (x - y);
+
+x:integer | y:integer | z:integer | w:integer
+5         | 3         | 25        | 12
+;
+
+// Test ROW field resolution - with functions
+rowFieldResolutionWithFunctions
+ROW a = 10, b = 3, c = ROUND(a / b, 2), d = c * 2;
+
+a:integer | b:integer | c:integer | d:integer
+10        | 3         | 3         | 6
+;
+
+// Test ROW field resolution - string concatenation
+rowFieldResolutionStringConcat
+ROW a = "Hello", b = "World", greeting = CONCAT(a, b);
+
+a:keyword   | b:keyword | greeting:keyword
+"Hello"     | "World"  | "HelloWorld"
+;
+
+// Test ROW field resolution - nested arithmetic
+rowFieldResolutionNestedArithmetic
+ROW a = 2, b = 3, c = 4, result = (a + b) * c - a;
+
+a:integer | b:integer | c:integer | result:integer
+2         | 3         | 4         | 18
+;
+
+// Test ROW field resolution - with null
+rowFieldResolutionWithNull
+ROW x = 10, y = null, z = x + y;
+
+x:integer | y:null | z:integer
+10        | null   | null
+;
+
+// Test ROW field resolution - field shadowing
+rowFieldResolutionShadowing
+required_capability: unique_names
+ROW x = 5, y = x * 2, x = y + 1;
+
+y:integer | x:integer
+10        | 11
+;
+
+rowFieldResolutionShadowing2
+required_capability: unique_names
+ROW x = 5, x = x * 2;
+
+x:integer
+10        
+;
+
+// Test ROW field resolution - mixed types
+rowFieldResolutionMixedTypes
+ROW num = 42, text = "Answer", combined = CONCAT(text, ": ", TO_STRING(num));
+
+num:integer | text:keyword | combined:keyword
+42          | "Answer"     | "Answer: 42"
+;
+
+// Test ROW field resolution - boolean expressions
+rowFieldResolutionBoolean
+ROW a = 10, b = 20, is_greater = b > a, is_equal = a == b;
+
+a:integer | b:integer | is_greater:boolean | is_equal:boolean
+10        | 20        | true               | false
+;
+
+// Test ROW field resolution - chained references
+rowFieldResolutionChained
+ROW a = 1, b = a + 1, c = b + 1, d = c + 1, e = d + 1;
+
+a:integer | b:integer | c:integer | d:integer | e:integer
+1         | 2         | 3         | 4         | 5
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

@@ -129,6 +129,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Rename;
+import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.TimeSeriesAggregate;
 import org.elasticsearch.xpack.esql.plan.logical.UnionAll;
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
@@ -499,6 +500,28 @@ private LocalRelation tableMapAsRelation(Source source, Map<String, Column> mapT
     }
 
     public static class ResolveRefs extends ParameterizedAnalyzerRule<LogicalPlan, AnalyzerContext> {
+
+        /**
+         * Override skipResolved to always process Row nodes, even when they are already resolved.
+         * <p>
+         * When all Row fields are literals (e.g., {@code ROW a = 1, b = 2, a = 3}), the Row node
+         * is resolved immediately after parsing since all expressions are resolved literals.
+         * However, we still need the Analyzer to:
+         * <ul>
+         *   <li>Handle duplicate field names (remove earlier definitions when later ones shadow them)</li>
+         *   <li>Resolve field references in later fields that refer to earlier ones
+         *       (e.g., {@code ROW x = 4, z = x + 1})</li>
+         * </ul>
+         * <p>
+         * In contrast, when Row contains field references (e.g., {@code ROW x = 4, z = x + y}),
+         * the Row is unresolved due to UnresolvedAttributes, so it would be processed anyway.
+         * This override ensures consistent handling for both cases.
+         */
+        @Override
+        protected boolean skipResolved(LogicalPlan plan) {
+            return plan instanceof Row == false;
+        }
+
         @Override
         protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
             if (plan.childrenResolved() == false) {
@@ -529,6 +552,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
                 case Fuse fuse -> resolveFuse(fuse, childrenOutput);
                 case Rerank r -> resolveRerank(r, childrenOutput);
                 case PromqlCommand promql -> resolvePromql(promql, childrenOutput);
+                case Row row -> resolveRow(row);
                 default -> plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput));
             };
         }
@@ -1185,6 +1209,57 @@ private LogicalPlan resolveEval(Eval eval, List<Attribute> childOutput) {
             return changed ? new Eval(eval.source(), eval.child(), newFields) : eval;
         }
 
+        /**
+         * Resolve Row fields, allowing later fields to reference earlier ones.
+         * Unlike resolveEval, Row fields are typically literals, so we directly substitute
+         * the literal expressions instead of creating attribute references.
+         * <p>
+         * For example:
+         * <pre>
+         * ROW x = 4, y = 2, z = x + y
+         * - x is resolved as literal 4
+         * - y is resolved as literal 2
+         * - z is resolved by substituting x with 4 and y with 2, resulting in 4 + 2
+         * </pre>
+         * <p>
+         * Field shadowing is supported:
+         * <pre>
+         * ROW x = 5, y = x * 2, x = y + 1
+         * - First x = 5 is defined
+         * - y = x * 2 resolves to y = 5 * 2 = 10
+         * - Second x = y + 1 resolves to x = 10 + 1 = 11
+         * - Final output: y = 10, x = 11 (first x is removed)
+         * </pre>
+         */
+        private LogicalPlan resolveRow(Row row) {
+            // Build a mapping from field names to their expressions for substitution
+            Map<String, Expression> fieldExpressions = new HashMap<>();
+            List<Alias> newFields = new ArrayList<>();
+            boolean changed = false;
+
+            for (Alias field : row.fields()) {
+                // Resolve unresolved attributes by substituting them with previously defined field expressions.
+                // If a matching field is found, replace the attribute with its expression; otherwise, keep it unresolved.
+                Alias result = (Alias) field.transformUp(UnresolvedAttribute.class, ua -> fieldExpressions.getOrDefault(ua.name(), ua));
+
+                changed |= result != field;
+
+                // Handle field shadowing: if a field with the same name already exists, remove it
+                // This ensures only the last definition of a field appears in the output
+                if (result.resolved()) {
+                    boolean removed = newFields.removeIf(existing -> existing.name().equals(result.name()));
+                    changed |= removed;
+
+                    // Store the field's child expression for future substitutions
+                    // If there's a duplicate name, the later one overrides
+                    fieldExpressions.put(result.name(), result.child());
+                }
+
+                newFields.add(result);
+            }
+            return changed ? new Row(row.source(), newFields) : row;
+        }
+
         /**
          * resolve each item manually.
          *

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java

@@ -47,12 +47,12 @@ public abstract static class ParameterizedAnalyzerRule<SubPlan extends LogicalPl
         // transformUp (post-order) - that is first children and then the node
         // but with a twist; only if the tree is not resolved or analyzed
         public final LogicalPlan apply(LogicalPlan plan, P context) {
-            return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t, context));
+            return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved(t) && t.resolved() ? t : rule(t, context));
         }
 
         protected abstract LogicalPlan rule(SubPlan plan, P context);
 
-        protected boolean skipResolved() {
+        protected boolean skipResolved(SubPlan plan) {
             return true;
         }
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java

@@ -110,7 +110,6 @@
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
-import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions;
 import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.esql.parser.ParserUtils.typedParsing;
 import static org.elasticsearch.xpack.esql.parser.ParserUtils.visitList;
@@ -347,10 +346,25 @@ public Map<String, Object> visitDissectCommandOptions(EsqlBaseParser.DissectComm
         return result;
     }
 
+    /**
+     * Creates a Row logical plan from the parsed ROW command.
+     * <p>
+     * Note: Unlike other commands, we do NOT call {@code mergeOutputExpressions} here to handle
+     * duplicate field names. This is intentional because:
+     * <ul>
+     *   <li>Row fields may contain forward references to earlier fields
+     *       (e.g., {@code ROW x = 4, y = 2, z = x + y})</li>
+     *   <li>Both duplicate field removal and field reference resolution are handled together
+     *       in the Analyzer's {@code resolveRow} method</li>
+     *   <li>This ensures consistent handling regardless of whether the Row contains only literals
+     *       (resolved at parse time) or includes field references (unresolved until analysis)</li>
+     * </ul>
+     *
+     * @see org.elasticsearch.xpack.esql.analysis.Analyzer.ResolveRefs#resolveRow
+     */
     @Override
-    @SuppressWarnings("unchecked")
     public LogicalPlan visitRowCommand(EsqlBaseParser.RowCommandContext ctx) {
-        return new Row(source(ctx), (List<Alias>) (List) mergeOutputExpressions(visitFields(ctx.fields()), List.of()));
+        return new Row(source(ctx), visitFields(ctx.fields()));
     }
 
     private LogicalPlan visitRelation(Source source, SourceCommand command, EsqlBaseParser.IndexPatternAndMetadataFieldsContext ctx) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -319,6 +319,50 @@ public void testRowAttributeResolution() {
         assertEquals(rowEmpNo.id(), empNo.id());
     }
 
+    public void testRowWithForwardReferences() {
+        EsIndex idx = EsIndexGenerator.esIndex("idx");
+        Analyzer analyzer = analyzer(IndexResolution.valid(idx));
+
+        // ROW x = 4, y = 2, z = x + y
+        var plan = analyzer.analyze(
+            new Row(
+                EMPTY,
+                List.of(
+                    new Alias(EMPTY, "x", new Literal(EMPTY, 4, INTEGER)),
+                    new Alias(EMPTY, "y", new Literal(EMPTY, 2, INTEGER)),
+                    new Alias(EMPTY, "z", new Add(EMPTY, new UnresolvedAttribute(EMPTY, "x"), new UnresolvedAttribute(EMPTY, "y")))
+                )
+            )
+        );
+
+        var limit = as(plan, Limit.class);
+        var row = as(limit.child(), Row.class);
+
+        assertEquals(3, row.fields().size());
+
+        // x should be literal 4
+        Alias xField = row.fields().get(0);
+        assertEquals("x", xField.name());
+        assertThat(xField.child(), instanceOf(Literal.class));
+        assertEquals(4, ((Literal) xField.child()).value());
+
+        // y should be literal 2
+        Alias yField = row.fields().get(1);
+        assertEquals("y", yField.name());
+        assertThat(yField.child(), instanceOf(Literal.class));
+        assertEquals(2, ((Literal) yField.child()).value());
+
+        // z should be Add(literal 4, literal 2), not Add(UnresolvedAttribute, UnresolvedAttribute)
+        Alias zField = row.fields().get(2);
+        assertEquals("z", zField.name());
+        assertThat(zField.child(), instanceOf(Add.class));
+        Add addExpr = (Add) zField.child();
+        assertThat(addExpr.left(), instanceOf(Literal.class));
+        assertThat(addExpr.right(), instanceOf(Literal.class));
+        assertEquals(4, ((Literal) addExpr.left()).value());
+        assertEquals(2, ((Literal) addExpr.right()).value());
+    }
+
     public void testUnresolvableAttribute() {
         Analyzer analyzer = analyzer(loadMapping("mapping-one-field.json", "idx"));