From 48824505cfc9c013d58cdefeb55830241a281fb1 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Sun, 8 Mar 2026 12:25:31 +0200 Subject: [PATCH 1/9] Implement a different approach to tracking "fields from index" for full-text functions limitations --- .../rest/generative/GenerativeRestTest.java | 246 +++++++++++++----- .../xpack/esql/generator/Column.java | 12 +- .../command/source/FromGenerator.java | 7 - .../function/FullTextFunctionGenerator.java | 48 +--- 4 files changed, 199 insertions(+), 114 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 465f29de89385..e8cc5a7a8682e 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.esql.generator.QueryExecuted; import org.elasticsearch.xpack.esql.generator.QueryExecutor; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; -import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; import org.junit.AfterClass; @@ -29,7 +29,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -45,7 +48,6 @@ import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_TYPE; import static org.elasticsearch.xpack.esql.generator.command.pipe.KeepGenerator.UNMAPPED_FIELD_NAMES; import static org.elasticsearch.xpack.esql.generator.command.source.FromGenerator.SET_UNMAPPED_FIELDS_PREFIX; -import static org.elasticsearch.xpack.esql.generator.command.source.FromGenerator.isFromSource; public abstract class GenerativeRestTest extends ESRestTestCase implements QueryExecutor { @@ -235,26 +237,22 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription : execute(previousResult.query() + command, previousResult.depth()); final boolean hasException = result.exception() != null; - if (hasException || checkResults(previousCommands, generator, current, previousResult, result).success() == false) { + if (hasException + || checkResults(previousCommands, generator, current, previousResult, result, currentSchema) + .success() == false) { if (hasException) { List commands = new ArrayList<>(previousCommands.size() + 1); commands.addAll(previousCommands); commands.add(current); - checkException(result, commands); + checkException(result, commands, currentSchema); } continueExecuting = false; currentSchema = List.of(); } else { continueExecuting = true; - currentSchema = result.outputSchema(); - } - if (previousCommands.isEmpty() && continueExecuting && isFromSource(current)) { - current.context() - .put( - FromGenerator.INDEX_FIELD_NAMES, - currentSchema.stream().map(Column::name).collect(java.util.stream.Collectors.toSet()) - ); + currentSchema = updateIndexMapped(result.outputSchema(), currentSchema, current); } + previousCommands.add(current); previousResult = result; } @@ -311,9 +309,15 @@ protected CommandGenerator sourceCommand() { return EsqlQueryGenerator.sourceCommand(); } - private record FailureContext(String errorMessage, String query, List previousCommands) { + private record FailureContext( + String errorMessage, + String query, + List previousCommands, + List currentSchema + ) { FailureContext { previousCommands = previousCommands == null ? List.of() : previousCommands; + currentSchema = currentSchema == null ? List.of() : currentSchema; } } @@ -329,7 +333,7 @@ private record FailureContext(String errorMessage, String query, List isScalarTypeMismatchError(ctx.errorMessage), ctx -> isFirstLastSameFieldError(ctx.errorMessage, ctx.query), ctx -> isForkOptimizationBugWithUnmappedFields(ctx.errorMessage, ctx.query), - ctx -> isFieldFullTextError(ctx.errorMessage, ctx.query, ctx.previousCommands), + ctx -> isFieldFullTextError(ctx.errorMessage, ctx.query, ctx.previousCommands, ctx.currentSchema), ctx -> isFullTextAfterSampleBug(ctx.errorMessage, ctx.query), ctx -> isFullTextAfterWhereBugs(ctx.errorMessage), ctx -> isLenientFalseFailedToCreateFullTextQueryError(ctx.errorMessage, ctx.query), }; @@ -351,7 +355,8 @@ protected static CommandGenerator.ValidationResult checkResults( CommandGenerator commandGenerator, CommandGenerator.CommandDescription commandDescription, QueryExecuted previousResult, - QueryExecuted result + QueryExecuted result, + List currentSchema ) { CommandGenerator.ValidationResult outputValidation = commandGenerator.validateOutput( previousCommands, @@ -362,7 +367,7 @@ protected static CommandGenerator.ValidationResult checkResults( result.result() ); if (outputValidation.success() == false) { - if (isAllowedFailure(new FailureContext(outputValidation.errorMessage(), result.query(), previousCommands))) { + if (isAllowedFailure(new FailureContext(outputValidation.errorMessage(), result.query(), previousCommands, currentSchema))) { return outputValidation; } fail("query: " + result.query() + "\nerror: " + outputValidation.errorMessage()); @@ -370,8 +375,12 @@ protected static CommandGenerator.ValidationResult checkResults( return outputValidation; } - protected void checkException(QueryExecuted query, List previousCommands) { - if (isAllowedFailure(new FailureContext(query.exception().getMessage(), query.query(), previousCommands))) { + protected void checkException( + QueryExecuted query, + List previousCommands, + List currentSchema + ) { + if (isAllowedFailure(new FailureContext(query.exception().getMessage(), query.query(), previousCommands, currentSchema))) { return; } fail("query: " + query.query() + "\nexception: " + query.exception().getMessage()); @@ -507,73 +516,178 @@ static boolean isForkOptimizationBugWithUnmappedFields(String errorMessage, Stri */ private static final Pattern DISSECT_GENERATED_FIELD_PATTERN = Pattern.compile("%\\{([^}]+)}"); - private static final Pattern MV_EXPAND_FIELD_PATTERN = Pattern.compile("(?i)\\|\\s*mv_expand\\s+`?([^`|\\s]+)`?"); - - private static final Pattern RENAME_NEW_FIELD_PATTERN = Pattern.compile("(?i)\\bas\\s+(`[^`]+`|[^,|\\s]+)"); + /** + * Captures both the source and target of a RENAME clause, e.g. {@code old_field AS new_field}. + * Group 1 is the source (possibly back-tick quoted), group 2 is the target. + */ + private static final Pattern RENAME_PAIR_PATTERN = Pattern.compile( + "\\s*(`[^`]+`|[^,\\s]+)\\s+[Aa][Ss]\\s+(`[^`]+`|[^,\\s]+)\\s*" + ); /** - * Checks if the error is a full-text function/operator rejecting a field that is not a FieldAttribute from an index mapping. It covers: + * Propagates the {@link Column#indexMapped()} flag through the pipeline after a command executes. + *

+ * The REST API does not expose attribute-type information, so this method infers it from: *

    - *
  • Fields added by an ENRICH command (enrich fields)
    • - *
  • Fields expanded by MV_EXPAND (the expanded fields)
    • - *
  • Fields created by GROK or DISSECT (the "extracted" fields)
    • - *
  • Fields renamed via RENAME
    • - *
  • Any query with a REGISTERED_DOMAIN command in the pipeline — its sub-fields (or fields derived from them - * via RENAME, EVAL, etc.) are not index mapping fields and may legitimately trigger this error
    • + *
  • The previous schema (columns carry their {@code indexMapped} status from earlier commands)
    • + *
  • The command description (command name and context — e.g. EVAL stores {@code NEW_COLUMNS})
    • *
- * The error is allowed only when the offending field can be traced back to one of these commands. + * Columns that are explicitly created by the command are marked {@code indexMapped=false}; + * columns that survive unchanged from the previous schema inherit their previous status. */ - static boolean isFieldFullTextError(String errorMessage, String query, List previousCommands) { + static List updateIndexMapped( + List newSchema, + List previousSchema, + CommandGenerator.CommandDescription command + ) { + if (newSchema == null || newSchema.isEmpty()) { + return newSchema; + } + if (previousSchema == null || previousSchema.isEmpty()) { + return newSchema; + } + + String commandName = command.commandName(); + if (commandName == null) { + return newSchema; + } + commandName = commandName.toLowerCase(Locale.ROOT); + + Map prevMapped = new HashMap<>(); + for (Column col : previousSchema) { + prevMapped.put(col.name(), col.indexMapped()); + } + + Set createdColumns = new HashSet<>(); + + switch (commandName) { + case "eval" -> { + Object newCols = command.context().get(EvalGenerator.NEW_COLUMNS); + if (newCols instanceof List list) { + list.forEach(name -> createdColumns.add((String) name)); + } + } + case "grok" -> { + Matcher gm = GROK_GENERATED_FIELD_PATTERN.matcher(command.commandString()); + while (gm.find()) { + createdColumns.add(EsqlQueryGenerator.unquote(gm.group(1))); + } + } + case "dissect" -> { + Matcher dm = DISSECT_GENERATED_FIELD_PATTERN.matcher(command.commandString()); + while (dm.find()) { + String generated = dm.group(1); + if (generated.startsWith("?") == false) { + createdColumns.add(EsqlQueryGenerator.unquote(generated)); + } + } + } + case "stats", "inline stats" -> { + return newSchema.stream() + .map(col -> new Column(col.name(), col.type(), col.originalTypes(), false)) + .toList(); + } + case "rename" -> { + return handleRenameIndexMapped(newSchema, prevMapped, command.commandString()); + } + case "registered_domain" -> { + String prefix = (String) command.context().get("prefix"); + if (prefix != null) { + for (String subField : List.of("domain", "registered_domain", "top_level_domain", "subdomain")) { + createdColumns.add(prefix + "." + subField); + } + } + } + case "uri_parts" -> { + String prefix = (String) command.context().get("prefix"); + if (prefix != null) { + return newSchema.stream().map(col -> { + if (col.name().startsWith(prefix + ".")) { + return new Column(col.name(), col.type(), col.originalTypes(), false); + } + Boolean prev = prevMapped.get(col.name()); + return new Column(col.name(), col.type(), col.originalTypes(), prev != null && prev); + }).toList(); + } + } + default -> { + // For commands that don't create named columns (KEEP, DROP, SORT, LIMIT, WHERE, etc.), + // any column not in previous is from the command (e.g. ENRICH, LOOKUP_JOIN, CHANGE_POINT) + } + } + + return newSchema.stream().map(col -> { + if (createdColumns.contains(col.name())) { + return new Column(col.name(), col.type(), col.originalTypes(), false); + } + Boolean prev = prevMapped.get(col.name()); + if (prev != null) { + return new Column(col.name(), col.type(), col.originalTypes(), prev); + } + return new Column(col.name(), col.type(), col.originalTypes(), false); + }).toList(); + } + + private static List handleRenameIndexMapped(List newSchema, Map prevMapped, String commandString) { + Map mapped = new HashMap<>(prevMapped); + String body = commandString.replaceFirst("(?i)^\\s*\\|\\s*rename\\s+", ""); + for (String pair : body.split(",")) { + Matcher m = RENAME_PAIR_PATTERN.matcher(pair); + if (m.matches()) { + String oldName = EsqlQueryGenerator.unquote(m.group(1).trim()); + String newName = EsqlQueryGenerator.unquote(m.group(2).trim()); + boolean wasMapped = mapped.getOrDefault(oldName, false); + mapped.remove(oldName); + mapped.put(newName, wasMapped); + } + } + return newSchema.stream().map(col -> { + Boolean isMapped = mapped.get(col.name()); + return new Column(col.name(), col.type(), col.originalTypes(), isMapped != null && isMapped); + }).toList(); + } + + /** + * Checks if the error is a full-text function/operator rejecting a field that is not from an index mapping. + * Uses the {@link Column#indexMapped()} flag from the current schema when available; falls back to + * command-history heuristics otherwise. + */ + static boolean isFieldFullTextError( + String errorMessage, + String query, + List previousCommands, + List currentSchema + ) { String errorWithoutLineBreaks = normalizeErrorMessage(errorMessage); Matcher m = NOT_A_FIELD_FROM_INDEX_PATTERN.matcher(errorWithoutLineBreaks); if (m.matches() == false) { return false; } - String lowerQuery = query.toLowerCase(java.util.Locale.ROOT); - if (lowerQuery.contains("| enrich ") || lowerQuery.startsWith("enrich ")) { - return true; - } - // see https://github.com/elastic/elasticsearch/issues/142713 String fieldName = EsqlQueryGenerator.unquote(m.group(1)); - Matcher mvMatcher = MV_EXPAND_FIELD_PATTERN.matcher(query); - while (mvMatcher.find()) { - if (EsqlQueryGenerator.unquote(mvMatcher.group(1)).equals(fieldName)) { - return true; + + if (currentSchema != null && currentSchema.isEmpty() == false) { + for (Column col : currentSchema) { + if (col.name().equals(fieldName)) { + return col.indexMapped() == false; + } } + // Field not found in schema — likely a pipeline artifact; allow the error + return true; } + // Fallback: blanket check for ENRICH + String lowerQuery = query.toLowerCase(Locale.ROOT); + if (lowerQuery.contains("| enrich ") || lowerQuery.startsWith("enrich ")) { + return true; + } + // Fallback: walk command history for (var previous : previousCommands) { String name = previous.commandName(); if (name == null) { continue; } - name = name.toLowerCase(java.util.Locale.ROOT); - if ("grok".equals(name)) { - Matcher gm = GROK_GENERATED_FIELD_PATTERN.matcher(previous.commandString()); - while (gm.find()) { - if (EsqlQueryGenerator.unquote(gm.group(1)).equals(fieldName)) { - return true; - } - } - } else if ("dissect".equals(name)) { - Matcher dm = DISSECT_GENERATED_FIELD_PATTERN.matcher(previous.commandString()); - while (dm.find()) { - String generated = dm.group(1); - if (generated.startsWith("?")) { - continue; - } - if (EsqlQueryGenerator.unquote(generated).equals(fieldName)) { - return true; - } - } - } else if ("rename".equals(name)) { - Matcher rm = RENAME_NEW_FIELD_PATTERN.matcher(previous.commandString()); - while (rm.find()) { - if (EsqlQueryGenerator.unquote(rm.group(1).trim()).equals(fieldName)) { - return true; - } - } - } else if ("registered_domain".equals(name)) { + name = name.toLowerCase(Locale.ROOT); + if ("registered_domain".equals(name) || "uri_parts".equals(name)) { return true; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java index 8159c920ded77..6db029e7db068 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java @@ -14,5 +14,15 @@ * @param name the field name * @param type the field type * @param originalTypes the original types, in case of a union type. + * @param indexMapped whether this column originates from an actual index mapping ({@code true}) or was + * computed/derived by a pipeline command such as EVAL, GROK, DISSECT, ENRICH, etc. ({@code false}). + * Full-text functions (match, match_phrase, multi_match, {@code :} operator) require index-mapped fields. */ -public record Column(String name, String type, List originalTypes) {} +public record Column(String name, String type, List originalTypes, boolean indexMapped) { + /** + * Backward-compatible constructor that defaults to {@code indexMapped = true}. + */ + public Column(String name, String type, List originalTypes) { + this(name, type, originalTypes, true); + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java index 536fbb3a06908..6e1105e735ddb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java @@ -29,13 +29,6 @@ public class FromGenerator implements CommandGenerator { */ public static final String UNMAPPED_FIELDS_ENABLED = "unmappedFieldsEnabled"; - /** - * Context key for the set of field names that come from the actual index mapping. - * Populated by the executor after the FROM command runs. - * Full-text functions that require FieldAttribute arguments use this to avoid computed columns. - */ - public static final String INDEX_FIELD_NAMES = "indexFieldNames"; - public static final String SET_UNMAPPED_FIELDS_PREFIX = "SET unmapped_fields=\"nullify\";"; /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java index 04bad2eecb127..83af796d01da9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java @@ -9,15 +9,11 @@ import org.elasticsearch.xpack.esql.generator.Column; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; -import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; -import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.test.ESTestCase.randomBoolean; @@ -68,48 +64,20 @@ private static boolean isQstrKqlAllowed(List indexFieldColumns(List columns, List previousCommands) { if (previousCommands == null || previousCommands.isEmpty()) { return null; } - Object stored = previousCommands.get(0).context().get(FromGenerator.INDEX_FIELD_NAMES); - if (stored instanceof Set == false) { + if (isFromSource(previousCommands.get(0)) == false) { return null; } - Set safeNames = new HashSet<>((Set) stored); - for (CommandGenerator.CommandDescription cmd : previousCommands) { - if ("eval".equals(cmd.commandName())) { - Object newCols = cmd.context().get(EvalGenerator.NEW_COLUMNS); - if (newCols instanceof List list) { - list.forEach(name -> safeNames.remove((String) name)); - } - } else if ("mv_expand".equals(cmd.commandName())) { - String expandedField = cmd.commandString().replaceFirst("(?i)^\\s*\\|\\s*mv_expand\\s+", "").trim(); - if (expandedField.startsWith("`") && expandedField.endsWith("`")) { - expandedField = expandedField.substring(1, expandedField.length() - 1); - } - safeNames.remove(expandedField); - } else if ("rename".equals(cmd.commandName())) { - String cmdStr = cmd.commandString().replaceFirst("(?i)^\\s*\\|\\s*rename\\s+", ""); - for (String pair : cmdStr.split(",")) { - Matcher m = RENAME_PAIR.matcher(pair); - if (m.matches()) { - String oldName = m.group(1); - String newName = m.group(2); - boolean wasSafe = safeNames.remove(oldName); - if (wasSafe) { - safeNames.add(newName); - } else { - safeNames.remove(newName); - } - } - } - } - } - return columns.stream().filter(c -> safeNames.contains(c.name())).toList(); + List result = columns.stream().filter(Column::indexMapped).toList(); + return result.isEmpty() ? null : result; } private static final Set MATCH_FIELD_TYPES = Set.of( From b7ce05b0cee236437e02c3fe45b35686f4287470 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 10 Mar 2026 15:10:15 +0200 Subject: [PATCH 2/9] Clean up GenerativeRestTest with closed issues --- muted-tests.yml | 6 --- .../rest/generative/GenerativeRestTest.java | 47 +------------------ 2 files changed, 2 insertions(+), 51 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 002c7ab923835..6162e653973ff 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -402,12 +402,6 @@ tests: - class: org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapperTests method: testRetrievedFromIdInTimeSeriesMode issue: https://github.com/elastic/elasticsearch/issues/143815 -- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/143023 -- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/143023 - class: org.elasticsearch.cluster.routing.allocation.allocator.NotPreferredAllocationRebalancingIT method: testAllocatorDoesNotMoveShardsToNotPreferredNode issue: https://github.com/elastic/elasticsearch/issues/143831 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index e8cc5a7a8682e..4edbcdbc1fd01 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -92,21 +92,12 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "unsupported logical plan node \\[Join\\]", // https://github.com/elastic/elasticsearch/issues/141978 "Unsupported right plan for lookup join \\[Eval\\]", // https://github.com/elastic/elasticsearch/issues/141870 "Does not support yet aggregations over constants", // https://github.com/elastic/elasticsearch/issues/118292 - "illegal data type \\[datetime\\]", // https://github.com/elastic/elasticsearch/issues/142137 - "Expected to replace a single StubRelation in the plan, but none found", // https://github.com/elastic/elasticsearch/issues/142219 - "blocks is empty", // https://github.com/elastic/elasticsearch/issues/142473 - "Overflow to represent absolute value of .*.MIN_VALUE", // https://github.com/elastic/elasticsearch/issues/142642 "found value \\[.*\\] type \\[unsupported\\]", // https://github.com/elastic/elasticsearch/issues/142761 - "illegal query_string option \\[boost\\]", // https://github.com/elastic/elasticsearch/issues/142758 "change point value \\[.*\\] must be numeric", // https://github.com/elastic/elasticsearch/issues/142858 - // https://github.com/elastic/elasticsearch/issues/142860 - "(Grok|Dissect) only supports KEYWORD or TEXT values, found expression \\[.*\\] type \\[NULL\\]", // https://github.com/elastic/elasticsearch/issues/142543 "Column \\[.*\\] has conflicting data types in FORK branches: \\[NULL\\] and \\[.*\\]", "Column \\[.*\\] has conflicting data types in FORK branches: \\[.*\\] and \\[NULL\\]", - "illegal match option \\[zero_terms_query\\]", // https://github.com/elastic/elasticsearch/issues/143070 "Field \\[.*\\] of type \\[.*\\] does not support match.* queries", - "Input for URI_PARTS must be of type [string] but is [null]", // https://github.com/elastic/elasticsearch/issues/143145 "JOIN left field \\[.*\\] of type \\[NULL\\] is incompatible with right", // https://github.com/elastic/elasticsearch/issues/141827 // https://github.com/elastic/elasticsearch/issues/141827 "JOIN left field \\[.*\\] of type \\[.*\\] is incompatible with right field \\[.*\\] of type \\[NULL\\]", @@ -180,19 +171,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query + "|aggregate_metric_double|dense_vector|tdigest|histogram|exponential_histogram|date_range)].*", Pattern.DOTALL ); - /** - * Matches FIRST(...) or LAST(...) function calls where the second argument is the literal {@code null}. - * See https://github.com/elastic/elasticsearch/issues/142180#issuecomment-3913054718 - */ - private static final Pattern FIRST_LAST_NULL_ARG_PATTERN = Pattern.compile("(?i)\\b(?:first|last)\\s*\\(.+?,\\s*null\\s*\\)"); - /** - * Matches FIRST(...) or LAST(...) function calls and captures both arguments. - * Used to detect when the same field is passed as both the search and sort parameters. - * See https://github.com/elastic/elasticsearch/issues/142180 - */ - private static final Pattern FIRST_LAST_CALL_PATTERN = Pattern.compile( - "(?i)\\b(?:first|last)\\s*\\(\\s*([^,()]+?)\\s*,\\s*([^,()]+?)\\s*\\)" - ); + private static final Set UNMAPPED_NAMES = Set.of(UNMAPPED_FIELD_NAMES); @Before @@ -252,7 +231,7 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription continueExecuting = true; currentSchema = updateIndexMapped(result.outputSchema(), currentSchema, current); } - + previousCommands.add(current); previousResult = result; } @@ -331,7 +310,6 @@ private record FailureContext( }, ctx -> isUnmappedFieldError(ctx.errorMessage, ctx.query), ctx -> isScalarTypeMismatchError(ctx.errorMessage), - ctx -> isFirstLastSameFieldError(ctx.errorMessage, ctx.query), ctx -> isForkOptimizationBugWithUnmappedFields(ctx.errorMessage, ctx.query), ctx -> isFieldFullTextError(ctx.errorMessage, ctx.query, ctx.previousCommands, ctx.currentSchema), ctx -> isFullTextAfterSampleBug(ctx.errorMessage, ctx.query), @@ -467,27 +445,6 @@ private static boolean isScalarTypeMismatchError(String errorMessage) { return SCALAR_TYPE_MISMATCH_PATTERN.matcher(errorWithoutLineBreaks).matches(); } - /** - * Checks if the error is an {@code ArrayIndexOutOfBoundsException} caused by calling FIRST or LAST with problematic arguments. - * See https://github.com/elastic/elasticsearch/issues/142180 - */ - private static boolean isFirstLastSameFieldError(String errorMessage, String query) { - String errorWithoutLineBreaks = normalizeErrorMessage(errorMessage); - if (errorWithoutLineBreaks.contains("out of bounds for length") == false) { - return false; - } - if (FIRST_LAST_NULL_ARG_PATTERN.matcher(query).find()) { - return true; - } - Matcher matcher = FIRST_LAST_CALL_PATTERN.matcher(query); - while (matcher.find()) { - if (matcher.group(1).equals(matcher.group(2))) { - return true; - } - } - return false; - } - private static final Pattern FORK_OPTIMIZED_INCORRECTLY_PATTERN = Pattern.compile( ".*Plan \\[.*\\] optimized incorrectly due to missing references \\[_fork.*", Pattern.DOTALL From e14d7e084d198f981e9df76745bfbfa8905e1ce0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 10 Mar 2026 13:30:36 +0000 Subject: [PATCH 3/9] [CI] Auto commit changes from spotless --- .../esql/qa/rest/generative/GenerativeRestTest.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 4edbcdbc1fd01..b854d93328b36 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -217,8 +217,7 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription final boolean hasException = result.exception() != null; if (hasException - || checkResults(previousCommands, generator, current, previousResult, result, currentSchema) - .success() == false) { + || checkResults(previousCommands, generator, current, previousResult, result, currentSchema).success() == false) { if (hasException) { List commands = new ArrayList<>(previousCommands.size() + 1); commands.addAll(previousCommands); @@ -477,9 +476,7 @@ static boolean isForkOptimizationBugWithUnmappedFields(String errorMessage, Stri * Captures both the source and target of a RENAME clause, e.g. {@code old_field AS new_field}. * Group 1 is the source (possibly back-tick quoted), group 2 is the target. */ - private static final Pattern RENAME_PAIR_PATTERN = Pattern.compile( - "\\s*(`[^`]+`|[^,\\s]+)\\s+[Aa][Ss]\\s+(`[^`]+`|[^,\\s]+)\\s*" - ); + private static final Pattern RENAME_PAIR_PATTERN = Pattern.compile("\\s*(`[^`]+`|[^,\\s]+)\\s+[Aa][Ss]\\s+(`[^`]+`|[^,\\s]+)\\s*"); /** * Propagates the {@link Column#indexMapped()} flag through the pipeline after a command executes. @@ -540,9 +537,7 @@ static List updateIndexMapped( } } case "stats", "inline stats" -> { - return newSchema.stream() - .map(col -> new Column(col.name(), col.type(), col.originalTypes(), false)) - .toList(); + return newSchema.stream().map(col -> new Column(col.name(), col.type(), col.originalTypes(), false)).toList(); } case "rename" -> { return handleRenameIndexMapped(newSchema, prevMapped, command.commandString()); From 9a37c5205dca22b6143c3b7975f5ec79af314326 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 11 Mar 2026 11:23:56 +0200 Subject: [PATCH 4/9] Address reviews --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 6 ++++++ .../java/org/elasticsearch/xpack/esql/generator/Column.java | 2 ++ 2 files changed, 8 insertions(+) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 4edbcdbc1fd01..17a65137a961f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -539,6 +539,12 @@ static List updateIndexMapped( } } } + case "mv_expand" -> { + String expanded = command.commandString().replaceFirst("(?i)^\\s*\\|\\s*mv_expand\\s+", "").trim(); + // Not truly a newly created column, but we need to override the indexMapped flag so that full-text functions don't use it. + // https://github.com/elastic/elasticsearch/issues/142713 + createdColumns.add(EsqlQueryGenerator.unquote(expanded)); + } case "stats", "inline stats" -> { return newSchema.stream() .map(col -> new Column(col.name(), col.type(), col.originalTypes(), false)) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java index 6db029e7db068..ad2700a5f494d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java @@ -17,6 +17,8 @@ * @param indexMapped whether this column originates from an actual index mapping ({@code true}) or was * computed/derived by a pipeline command such as EVAL, GROK, DISSECT, ENRICH, etc. ({@code false}). * Full-text functions (match, match_phrase, multi_match, {@code :} operator) require index-mapped fields. + * An exception here is MV_EXPAND which, even if has as ouput a field from an index, this field is forbidden to be used + * in full-text functions, as well. See https://github.com/elastic/elasticsearch/issues/142713 */ public record Column(String name, String type, List originalTypes, boolean indexMapped) { /** From ba78f8e9a5285965af728553bd218af34f6f6ed1 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 11 Mar 2026 11:34:16 +0200 Subject: [PATCH 5/9] Clean up code --- .../rest/generative/GenerativeRestTest.java | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 17a65137a961f..29bdecad65ec6 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -46,6 +46,7 @@ import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_NAME; import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_ORIGINAL_TYPES; import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_TYPE; +import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.unquote; import static org.elasticsearch.xpack.esql.generator.command.pipe.KeepGenerator.UNMAPPED_FIELD_NAMES; import static org.elasticsearch.xpack.esql.generator.command.source.FromGenerator.SET_UNMAPPED_FIELDS_PREFIX; @@ -527,7 +528,7 @@ static List updateIndexMapped( case "grok" -> { Matcher gm = GROK_GENERATED_FIELD_PATTERN.matcher(command.commandString()); while (gm.find()) { - createdColumns.add(EsqlQueryGenerator.unquote(gm.group(1))); + createdColumns.add(unquote(gm.group(1))); } } case "dissect" -> { @@ -535,7 +536,7 @@ static List updateIndexMapped( while (dm.find()) { String generated = dm.group(1); if (generated.startsWith("?") == false) { - createdColumns.add(EsqlQueryGenerator.unquote(generated)); + createdColumns.add(unquote(generated)); } } } @@ -543,7 +544,7 @@ static List updateIndexMapped( String expanded = command.commandString().replaceFirst("(?i)^\\s*\\|\\s*mv_expand\\s+", "").trim(); // Not truly a newly created column, but we need to override the indexMapped flag so that full-text functions don't use it. // https://github.com/elastic/elasticsearch/issues/142713 - createdColumns.add(EsqlQueryGenerator.unquote(expanded)); + createdColumns.add(unquote(expanded)); } case "stats", "inline stats" -> { return newSchema.stream() @@ -564,13 +565,11 @@ static List updateIndexMapped( case "uri_parts" -> { String prefix = (String) command.context().get("prefix"); if (prefix != null) { - return newSchema.stream().map(col -> { + for (Column col : newSchema) { if (col.name().startsWith(prefix + ".")) { - return new Column(col.name(), col.type(), col.originalTypes(), false); + createdColumns.add(col.name()); } - Boolean prev = prevMapped.get(col.name()); - return new Column(col.name(), col.type(), col.originalTypes(), prev != null && prev); - }).toList(); + } } } default -> { @@ -579,7 +578,11 @@ static List updateIndexMapped( } } - return newSchema.stream().map(col -> { + return applyIndexMapped(newSchema, createdColumns, prevMapped); + } + + private static List applyIndexMapped(List schema, Set createdColumns, Map prevMapped) { + return schema.stream().map(col -> { if (createdColumns.contains(col.name())) { return new Column(col.name(), col.type(), col.originalTypes(), false); } @@ -597,8 +600,8 @@ private static List handleRenameIndexMapped(List newSchema, Map< for (String pair : body.split(",")) { Matcher m = RENAME_PAIR_PATTERN.matcher(pair); if (m.matches()) { - String oldName = EsqlQueryGenerator.unquote(m.group(1).trim()); - String newName = EsqlQueryGenerator.unquote(m.group(2).trim()); + String oldName = unquote(m.group(1).trim()); + String newName = unquote(m.group(2).trim()); boolean wasMapped = mapped.getOrDefault(oldName, false); mapped.remove(oldName); mapped.put(newName, wasMapped); @@ -626,7 +629,7 @@ static boolean isFieldFullTextError( if (m.matches() == false) { return false; } - String fieldName = EsqlQueryGenerator.unquote(m.group(1)); + String fieldName = unquote(m.group(1)); if (currentSchema != null && currentSchema.isEmpty() == false) { for (Column col : currentSchema) { @@ -657,17 +660,22 @@ static boolean isFieldFullTextError( return false; } + private static final Pattern FULL_TEXT_AFTER_SAMPLE_PATTERN = Pattern.compile( + ".*\\[(KQL|QSTR)] function cannot be used after SAMPLE.*", + Pattern.DOTALL + ); + /** * SAMPLE should not block QSTR/KQL when it appears after the WHERE containing them, but currently it does. * See https://github.com/elastic/elasticsearch/issues/142694 */ static boolean isFullTextAfterSampleBug(String errorMessage, String query) { return FULL_TEXT_AFTER_SAMPLE_PATTERN.matcher(normalizeErrorMessage(errorMessage)).matches() - && query.toLowerCase(java.util.Locale.ROOT).contains("| sample"); + && query.toLowerCase(Locale.ROOT).contains("| sample"); } - private static final Pattern FULL_TEXT_AFTER_SAMPLE_PATTERN = Pattern.compile( - ".*\\[(KQL|QSTR)] function cannot be used after SAMPLE.*", + private static final Pattern FULL_TEXT_AFTER_WHERE_PATTERN = Pattern.compile( + ".*(?:(?:\\[(?:KQL|QSTR|MATCH|MultiMatch|MatchPhrase)] function)|(?:\\[:\\] operator)) cannot be used after \\(?WHERE.*", Pattern.DOTALL ); @@ -680,9 +688,14 @@ static boolean isFullTextAfterWhereBugs(String errorMessage) { return FULL_TEXT_AFTER_WHERE_PATTERN.matcher(errorWithoutLineBreaks).matches(); } - private static final Pattern FULL_TEXT_AFTER_WHERE_PATTERN = Pattern.compile( - ".*(?:(?:\\[(?:KQL|QSTR|MATCH|MultiMatch|MatchPhrase)] function)|(?:\\[:\\] operator)) cannot be used after \\(?WHERE.*", - Pattern.DOTALL + private static final Pattern MULTI_MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bmulti_match\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" + ); + private static final Pattern MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bmatch\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" + ); + private static final Pattern QSTR_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bqstr\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" ); /** @@ -698,16 +711,6 @@ static boolean isLenientFalseFailedToCreateFullTextQueryError(String errorMessag || QSTR_LENIENT_FALSE_PATTERN.matcher(query).find(); } - private static final Pattern MULTI_MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bmulti_match\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - private static final Pattern MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bmatch\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - private static final Pattern QSTR_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bqstr\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - @Override @SuppressWarnings("unchecked") public QueryExecuted execute(String query, int depth) { @@ -738,7 +741,7 @@ private static List outputSchema(Map a) { } return cols.stream() .map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))) - .collect(Collectors.toList()); + .toList(); } @SuppressWarnings("unchecked") From 39575df16e3411cf11141c5a737380da2af85409 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 11 Mar 2026 11:42:37 +0200 Subject: [PATCH 6/9] More clean up --- .../qa/rest/generative/GenerativeRestTest.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 29bdecad65ec6..eae03f77996e8 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -640,23 +640,6 @@ static boolean isFieldFullTextError( // Field not found in schema — likely a pipeline artifact; allow the error return true; } - - // Fallback: blanket check for ENRICH - String lowerQuery = query.toLowerCase(Locale.ROOT); - if (lowerQuery.contains("| enrich ") || lowerQuery.startsWith("enrich ")) { - return true; - } - // Fallback: walk command history - for (var previous : previousCommands) { - String name = previous.commandName(); - if (name == null) { - continue; - } - name = name.toLowerCase(Locale.ROOT); - if ("registered_domain".equals(name) || "uri_parts".equals(name)) { - return true; - } - } return false; } From 12c0702c3fc12f9c5102bfc2765f9febe975de1f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 11 Mar 2026 09:54:07 +0000 Subject: [PATCH 7/9] [CI] Auto commit changes from spotless --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index edab79dbf2491..8cb1aaf578e58 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -717,9 +717,7 @@ private static List outputSchema(Map a) { if (cols == null) { return null; } - return cols.stream() - .map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))) - .toList(); + return cols.stream().map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))).toList(); } @SuppressWarnings("unchecked") From da9409df0dd918d3317a144c734f6b6bd7e4d0ea Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 11 Mar 2026 19:14:30 +0200 Subject: [PATCH 8/9] I hoped this would be fixed in time. Putting the allowed error back --- .../xpack/esql/qa/rest/generative/GenerativeRestTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index edab79dbf2491..e909d8e670d25 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -95,6 +95,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "Does not support yet aggregations over constants", // https://github.com/elastic/elasticsearch/issues/118292 "found value \\[.*\\] type \\[unsupported\\]", // https://github.com/elastic/elasticsearch/issues/142761 "change point value \\[.*\\] must be numeric", // https://github.com/elastic/elasticsearch/issues/142858 + "illegal query_string option \\[boost\\]", // https://github.com/elastic/elasticsearch/issues/142758 // https://github.com/elastic/elasticsearch/issues/142543 "Column \\[.*\\] has conflicting data types in FORK branches: \\[NULL\\] and \\[.*\\]", "Column \\[.*\\] has conflicting data types in FORK branches: \\[.*\\] and \\[NULL\\]", From ea2ee7d68affcd7492bf1d9837941c0e2efab0f6 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 12 Mar 2026 10:41:35 +0200 Subject: [PATCH 9/9] Track the enrich fields as well --- .../rest/generative/GenerativeRestTest.java | 11 +++++- .../command/pipe/EnrichGenerator.java | 34 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 008a4ae915416..a8acebc860df1 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.generator.QueryExecuted; import org.elasticsearch.xpack.esql.generator.QueryExecutor; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EnrichGenerator; import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; @@ -568,9 +569,17 @@ static List updateIndexMapped( } } } + case "enrich" -> { + // Enrich fields can shadow existing index columns, so we use the policy's declared enrich_fields + // from the context to ensure they are marked as non-index-mapped even when names collide. + Object enrichFieldsObj = command.context().get(EnrichGenerator.ENRICH_FIELDS); + if (enrichFieldsObj instanceof List enrichFieldsList) { + enrichFieldsList.forEach(name -> createdColumns.add((String) name)); + } + } default -> { // For commands that don't create named columns (KEEP, DROP, SORT, LIMIT, WHERE, etc.), - // any column not in previous is from the command (e.g. ENRICH, LOOKUP_JOIN, CHANGE_POINT) + // any column not in previous is from the command (e.g. LOOKUP_JOIN, CHANGE_POINT) } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java index be3d547cf535c..89913a9cd2014 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java @@ -7,21 +7,28 @@ package org.elasticsearch.xpack.esql.generator.command.pipe; +import org.elasticsearch.xpack.esql.CsvTestsDataLoader; import org.elasticsearch.xpack.esql.generator.Column; import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; import org.elasticsearch.xpack.esql.generator.QueryExecutor; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.elasticsearch.test.ESTestCase.randomFrom; public class EnrichGenerator implements CommandGenerator { public static final String ENRICH = "enrich"; + public static final String ENRICH_FIELDS = "enrichFields"; public static final CommandGenerator INSTANCE = new EnrichGenerator(); + private static final Pattern ENRICH_FIELDS_PATTERN = Pattern.compile("\"enrich_fields\"\\s*:\\s*\\[([^\\]]*)]"); + @Override public CommandDescription generate( List previousCommands, @@ -35,11 +42,28 @@ public CommandDescription generate( } // TODO add WITH - String cmdString = " | enrich " - + randomFrom(EsqlQueryGenerator.policiesOnKeyword(schema.enrichPolicies())).policyName() - + " on " - + field; - return new CommandDescription(ENRICH, this, cmdString, Map.of()); + CsvTestsDataLoader.EnrichConfig policy = randomFrom(EsqlQueryGenerator.policiesOnKeyword(schema.enrichPolicies())); + String cmdString = " | enrich " + policy.policyName() + " on " + field; + List enrichFields = parseEnrichFields(policy.loadPolicy()); + return new CommandDescription(ENRICH, this, cmdString, Map.of(ENRICH_FIELDS, enrichFields)); + } + + static List parseEnrichFields(String policyJson) { + Matcher m = ENRICH_FIELDS_PATTERN.matcher(policyJson); + if (m.find()) { + List fields = new ArrayList<>(); + for (String part : m.group(1).split(",")) { + String trimmed = part.trim(); + if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) { + trimmed = trimmed.substring(1, trimmed.length() - 1); + } + if (trimmed.isEmpty() == false) { + fields.add(trimmed); + } + } + return fields; + } + return List.of(); } @Override