diff --git a/muted-tests.yml b/muted-tests.yml index 70a239844d732..e5005697fa433 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -378,12 +378,6 @@ tests: - class: org.elasticsearch.upgrades.FullClusterRestartIT method: testOperationBasedRecovery {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/143802 -- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/143023 -- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT - method: test - issue: https://github.com/elastic/elasticsearch/issues/143023 - class: org.elasticsearch.cluster.routing.allocation.allocator.NotPreferredAllocationRebalancingIT method: testAllocatorDoesNotMoveShardsToNotPreferredNode issue: https://github.com/elastic/elasticsearch/issues/143831 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 465f29de89385..a8acebc860df1 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -19,7 +19,8 @@ import org.elasticsearch.xpack.esql.generator.QueryExecuted; import org.elasticsearch.xpack.esql.generator.QueryExecutor; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; -import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EnrichGenerator; +import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; import org.junit.AfterClass; @@ -29,7 +30,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; @@ -43,9 +47,9 @@ import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_NAME; import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_ORIGINAL_TYPES; import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.COLUMN_TYPE; +import static org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator.unquote; import static org.elasticsearch.xpack.esql.generator.command.pipe.KeepGenerator.UNMAPPED_FIELD_NAMES; import static org.elasticsearch.xpack.esql.generator.command.source.FromGenerator.SET_UNMAPPED_FIELDS_PREFIX; -import static org.elasticsearch.xpack.esql.generator.command.source.FromGenerator.isFromSource; public abstract class GenerativeRestTest extends ESRestTestCase implements QueryExecutor { @@ -90,21 +94,13 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "unsupported logical plan node \\[Join\\]", // https://github.com/elastic/elasticsearch/issues/141978 "Unsupported right plan for lookup join \\[Eval\\]", // https://github.com/elastic/elasticsearch/issues/141870 "Does not support yet aggregations over constants", // https://github.com/elastic/elasticsearch/issues/118292 - "illegal data type \\[datetime\\]", // https://github.com/elastic/elasticsearch/issues/142137 - "Expected to replace a single StubRelation in the plan, but none found", // https://github.com/elastic/elasticsearch/issues/142219 - "blocks is empty", // https://github.com/elastic/elasticsearch/issues/142473 - "Overflow to represent absolute value of .*.MIN_VALUE", // https://github.com/elastic/elasticsearch/issues/142642 "found value \\[.*\\] type \\[unsupported\\]", // https://github.com/elastic/elasticsearch/issues/142761 - "illegal query_string option \\[boost\\]", // https://github.com/elastic/elasticsearch/issues/142758 "change point value \\[.*\\] must be numeric", // https://github.com/elastic/elasticsearch/issues/142858 - // https://github.com/elastic/elasticsearch/issues/142860 - "(Grok|Dissect) only supports KEYWORD or TEXT values, found expression \\[.*\\] type \\[NULL\\]", + "illegal query_string option \\[boost\\]", // https://github.com/elastic/elasticsearch/issues/142758 // https://github.com/elastic/elasticsearch/issues/142543 "Column \\[.*\\] has conflicting data types in FORK branches: \\[NULL\\] and \\[.*\\]", "Column \\[.*\\] has conflicting data types in FORK branches: \\[.*\\] and \\[NULL\\]", - "illegal match option \\[zero_terms_query\\]", // https://github.com/elastic/elasticsearch/issues/143070 "Field \\[.*\\] of type \\[.*\\] does not support match.* queries", - "Input for URI_PARTS must be of type [string] but is [null]", // https://github.com/elastic/elasticsearch/issues/143145 "JOIN left field \\[.*\\] of type \\[NULL\\] is incompatible with right", // https://github.com/elastic/elasticsearch/issues/141827 // https://github.com/elastic/elasticsearch/issues/141827 "JOIN left field \\[.*\\] of type \\[.*\\] is incompatible with right field \\[.*\\] of type \\[NULL\\]", @@ -178,19 +174,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query + "|aggregate_metric_double|dense_vector|tdigest|histogram|exponential_histogram|date_range)].*", Pattern.DOTALL ); - /** - * Matches FIRST(...) or LAST(...) function calls where the second argument is the literal {@code null}. - * See https://github.com/elastic/elasticsearch/issues/142180#issuecomment-3913054718 - */ - private static final Pattern FIRST_LAST_NULL_ARG_PATTERN = Pattern.compile("(?i)\\b(?:first|last)\\s*\\(.+?,\\s*null\\s*\\)"); - /** - * Matches FIRST(...) or LAST(...) function calls and captures both arguments. - * Used to detect when the same field is passed as both the search and sort parameters. - * See https://github.com/elastic/elasticsearch/issues/142180 - */ - private static final Pattern FIRST_LAST_CALL_PATTERN = Pattern.compile( - "(?i)\\b(?:first|last)\\s*\\(\\s*([^,()]+?)\\s*,\\s*([^,()]+?)\\s*\\)" - ); + private static final Set UNMAPPED_NAMES = Set.of(UNMAPPED_FIELD_NAMES); @Before @@ -235,26 +219,21 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription : execute(previousResult.query() + command, previousResult.depth()); final boolean hasException = result.exception() != null; - if (hasException || checkResults(previousCommands, generator, current, previousResult, result).success() == false) { + if (hasException + || checkResults(previousCommands, generator, current, previousResult, result, currentSchema).success() == false) { if (hasException) { List commands = new ArrayList<>(previousCommands.size() + 1); commands.addAll(previousCommands); commands.add(current); - checkException(result, commands); + checkException(result, commands, currentSchema); } continueExecuting = false; currentSchema = List.of(); } else { continueExecuting = true; - currentSchema = result.outputSchema(); - } - if (previousCommands.isEmpty() && continueExecuting && isFromSource(current)) { - current.context() - .put( - FromGenerator.INDEX_FIELD_NAMES, - currentSchema.stream().map(Column::name).collect(java.util.stream.Collectors.toSet()) - ); + currentSchema = updateIndexMapped(result.outputSchema(), currentSchema, current); } + previousCommands.add(current); previousResult = result; } @@ -311,9 +290,15 @@ protected CommandGenerator sourceCommand() { return EsqlQueryGenerator.sourceCommand(); } - private record FailureContext(String errorMessage, String query, List previousCommands) { + private record FailureContext( + String errorMessage, + String query, + List previousCommands, + List currentSchema + ) { FailureContext { previousCommands = previousCommands == null ? List.of() : previousCommands; + currentSchema = currentSchema == null ? List.of() : currentSchema; } } @@ -327,9 +312,8 @@ private record FailureContext(String errorMessage, String query, List isUnmappedFieldError(ctx.errorMessage, ctx.query), ctx -> isScalarTypeMismatchError(ctx.errorMessage), - ctx -> isFirstLastSameFieldError(ctx.errorMessage, ctx.query), ctx -> isForkOptimizationBugWithUnmappedFields(ctx.errorMessage, ctx.query), - ctx -> isFieldFullTextError(ctx.errorMessage, ctx.query, ctx.previousCommands), + ctx -> isFieldFullTextError(ctx.errorMessage, ctx.query, ctx.previousCommands, ctx.currentSchema), ctx -> isFullTextAfterSampleBug(ctx.errorMessage, ctx.query), ctx -> isFullTextAfterWhereBugs(ctx.errorMessage), ctx -> isLenientFalseFailedToCreateFullTextQueryError(ctx.errorMessage, ctx.query), }; @@ -351,7 +335,8 @@ protected static CommandGenerator.ValidationResult checkResults( CommandGenerator commandGenerator, CommandGenerator.CommandDescription commandDescription, QueryExecuted previousResult, - QueryExecuted result + QueryExecuted result, + List currentSchema ) { CommandGenerator.ValidationResult outputValidation = commandGenerator.validateOutput( previousCommands, @@ -362,7 +347,7 @@ protected static CommandGenerator.ValidationResult checkResults( result.result() ); if (outputValidation.success() == false) { - if (isAllowedFailure(new FailureContext(outputValidation.errorMessage(), result.query(), previousCommands))) { + if (isAllowedFailure(new FailureContext(outputValidation.errorMessage(), result.query(), previousCommands, currentSchema))) { return outputValidation; } fail("query: " + result.query() + "\nerror: " + outputValidation.errorMessage()); @@ -370,8 +355,12 @@ protected static CommandGenerator.ValidationResult checkResults( return outputValidation; } - protected void checkException(QueryExecuted query, List previousCommands) { - if (isAllowedFailure(new FailureContext(query.exception().getMessage(), query.query(), previousCommands))) { + protected void checkException( + QueryExecuted query, + List previousCommands, + List currentSchema + ) { + if (isAllowedFailure(new FailureContext(query.exception().getMessage(), query.query(), previousCommands, currentSchema))) { return; } fail("query: " + query.query() + "\nexception: " + query.exception().getMessage()); @@ -458,27 +447,6 @@ private static boolean isScalarTypeMismatchError(String errorMessage) { return SCALAR_TYPE_MISMATCH_PATTERN.matcher(errorWithoutLineBreaks).matches(); } - /** - * Checks if the error is an {@code ArrayIndexOutOfBoundsException} caused by calling FIRST or LAST with problematic arguments. - * See https://github.com/elastic/elasticsearch/issues/142180 - */ - private static boolean isFirstLastSameFieldError(String errorMessage, String query) { - String errorWithoutLineBreaks = normalizeErrorMessage(errorMessage); - if (errorWithoutLineBreaks.contains("out of bounds for length") == false) { - return false; - } - if (FIRST_LAST_NULL_ARG_PATTERN.matcher(query).find()) { - return true; - } - Matcher matcher = FIRST_LAST_CALL_PATTERN.matcher(query); - while (matcher.find()) { - if (matcher.group(1).equals(matcher.group(2))) { - return true; - } - } - return false; - } - private static final Pattern FORK_OPTIMIZED_INCORRECTLY_PATTERN = Pattern.compile( ".*Plan \\[.*\\] optimized incorrectly due to missing references \\[_fork.*", Pattern.DOTALL @@ -507,90 +475,195 @@ static boolean isForkOptimizationBugWithUnmappedFields(String errorMessage, Stri */ private static final Pattern DISSECT_GENERATED_FIELD_PATTERN = Pattern.compile("%\\{([^}]+)}"); - private static final Pattern MV_EXPAND_FIELD_PATTERN = Pattern.compile("(?i)\\|\\s*mv_expand\\s+`?([^`|\\s]+)`?"); - - private static final Pattern RENAME_NEW_FIELD_PATTERN = Pattern.compile("(?i)\\bas\\s+(`[^`]+`|[^,|\\s]+)"); + /** + * Captures both the source and target of a RENAME clause, e.g. {@code old_field AS new_field}. + * Group 1 is the source (possibly back-tick quoted), group 2 is the target. + */ + private static final Pattern RENAME_PAIR_PATTERN = Pattern.compile("\\s*(`[^`]+`|[^,\\s]+)\\s+[Aa][Ss]\\s+(`[^`]+`|[^,\\s]+)\\s*"); /** - * Checks if the error is a full-text function/operator rejecting a field that is not a FieldAttribute from an index mapping. It covers: + * Propagates the {@link Column#indexMapped()} flag through the pipeline after a command executes. + *

+ * The REST API does not expose attribute-type information, so this method infers it from: *

    - *
  • Fields added by an ENRICH command (enrich fields)
    • - *
  • Fields expanded by MV_EXPAND (the expanded fields)
    • - *
  • Fields created by GROK or DISSECT (the "extracted" fields)
    • - *
  • Fields renamed via RENAME
    • - *
  • Any query with a REGISTERED_DOMAIN command in the pipeline — its sub-fields (or fields derived from them - * via RENAME, EVAL, etc.) are not index mapping fields and may legitimately trigger this error
    • + *
  • The previous schema (columns carry their {@code indexMapped} status from earlier commands)
    • + *
  • The command description (command name and context — e.g. EVAL stores {@code NEW_COLUMNS})
    • *
- * The error is allowed only when the offending field can be traced back to one of these commands. + * Columns that are explicitly created by the command are marked {@code indexMapped=false}; + * columns that survive unchanged from the previous schema inherit their previous status. */ - static boolean isFieldFullTextError(String errorMessage, String query, List previousCommands) { - String errorWithoutLineBreaks = normalizeErrorMessage(errorMessage); - Matcher m = NOT_A_FIELD_FROM_INDEX_PATTERN.matcher(errorWithoutLineBreaks); - if (m.matches() == false) { - return false; + static List updateIndexMapped( + List newSchema, + List previousSchema, + CommandGenerator.CommandDescription command + ) { + if (newSchema == null || newSchema.isEmpty()) { + return newSchema; } - String lowerQuery = query.toLowerCase(java.util.Locale.ROOT); - if (lowerQuery.contains("| enrich ") || lowerQuery.startsWith("enrich ")) { - return true; + if (previousSchema == null || previousSchema.isEmpty()) { + return newSchema; } - // see https://github.com/elastic/elasticsearch/issues/142713 - String fieldName = EsqlQueryGenerator.unquote(m.group(1)); - Matcher mvMatcher = MV_EXPAND_FIELD_PATTERN.matcher(query); - while (mvMatcher.find()) { - if (EsqlQueryGenerator.unquote(mvMatcher.group(1)).equals(fieldName)) { - return true; - } + + String commandName = command.commandName(); + if (commandName == null) { + return newSchema; } + commandName = commandName.toLowerCase(Locale.ROOT); + + Map prevMapped = new HashMap<>(); + for (Column col : previousSchema) { + prevMapped.put(col.name(), col.indexMapped()); + } + + Set createdColumns = new HashSet<>(); - for (var previous : previousCommands) { - String name = previous.commandName(); - if (name == null) { - continue; + switch (commandName) { + case "eval" -> { + Object newCols = command.context().get(EvalGenerator.NEW_COLUMNS); + if (newCols instanceof List list) { + list.forEach(name -> createdColumns.add((String) name)); + } } - name = name.toLowerCase(java.util.Locale.ROOT); - if ("grok".equals(name)) { - Matcher gm = GROK_GENERATED_FIELD_PATTERN.matcher(previous.commandString()); + case "grok" -> { + Matcher gm = GROK_GENERATED_FIELD_PATTERN.matcher(command.commandString()); while (gm.find()) { - if (EsqlQueryGenerator.unquote(gm.group(1)).equals(fieldName)) { - return true; - } + createdColumns.add(unquote(gm.group(1))); } - } else if ("dissect".equals(name)) { - Matcher dm = DISSECT_GENERATED_FIELD_PATTERN.matcher(previous.commandString()); + } + case "dissect" -> { + Matcher dm = DISSECT_GENERATED_FIELD_PATTERN.matcher(command.commandString()); while (dm.find()) { String generated = dm.group(1); - if (generated.startsWith("?")) { - continue; + if (generated.startsWith("?") == false) { + createdColumns.add(unquote(generated)); } - if (EsqlQueryGenerator.unquote(generated).equals(fieldName)) { - return true; + } + } + case "mv_expand" -> { + String expanded = command.commandString().replaceFirst("(?i)^\\s*\\|\\s*mv_expand\\s+", "").trim(); + // Not truly a newly created column, but we need to override the indexMapped flag so that full-text functions don't use it. + // https://github.com/elastic/elasticsearch/issues/142713 + createdColumns.add(unquote(expanded)); + } + case "stats", "inline stats" -> { + return newSchema.stream().map(col -> new Column(col.name(), col.type(), col.originalTypes(), false)).toList(); + } + case "rename" -> { + return handleRenameIndexMapped(newSchema, prevMapped, command.commandString()); + } + case "registered_domain" -> { + String prefix = (String) command.context().get("prefix"); + if (prefix != null) { + for (String subField : List.of("domain", "registered_domain", "top_level_domain", "subdomain")) { + createdColumns.add(prefix + "." + subField); } } - } else if ("rename".equals(name)) { - Matcher rm = RENAME_NEW_FIELD_PATTERN.matcher(previous.commandString()); - while (rm.find()) { - if (EsqlQueryGenerator.unquote(rm.group(1).trim()).equals(fieldName)) { - return true; + } + case "uri_parts" -> { + String prefix = (String) command.context().get("prefix"); + if (prefix != null) { + for (Column col : newSchema) { + if (col.name().startsWith(prefix + ".")) { + createdColumns.add(col.name()); + } } } - } else if ("registered_domain".equals(name)) { - return true; + } + case "enrich" -> { + // Enrich fields can shadow existing index columns, so we use the policy's declared enrich_fields + // from the context to ensure they are marked as non-index-mapped even when names collide. + Object enrichFieldsObj = command.context().get(EnrichGenerator.ENRICH_FIELDS); + if (enrichFieldsObj instanceof List enrichFieldsList) { + enrichFieldsList.forEach(name -> createdColumns.add((String) name)); + } + } + default -> { + // For commands that don't create named columns (KEEP, DROP, SORT, LIMIT, WHERE, etc.), + // any column not in previous is from the command (e.g. LOOKUP_JOIN, CHANGE_POINT) } } + + return applyIndexMapped(newSchema, createdColumns, prevMapped); + } + + private static List applyIndexMapped(List schema, Set createdColumns, Map prevMapped) { + return schema.stream().map(col -> { + if (createdColumns.contains(col.name())) { + return new Column(col.name(), col.type(), col.originalTypes(), false); + } + Boolean prev = prevMapped.get(col.name()); + if (prev != null) { + return new Column(col.name(), col.type(), col.originalTypes(), prev); + } + return new Column(col.name(), col.type(), col.originalTypes(), false); + }).toList(); + } + + private static List handleRenameIndexMapped(List newSchema, Map prevMapped, String commandString) { + Map mapped = new HashMap<>(prevMapped); + String body = commandString.replaceFirst("(?i)^\\s*\\|\\s*rename\\s+", ""); + for (String pair : body.split(",")) { + Matcher m = RENAME_PAIR_PATTERN.matcher(pair); + if (m.matches()) { + String oldName = unquote(m.group(1).trim()); + String newName = unquote(m.group(2).trim()); + boolean wasMapped = mapped.getOrDefault(oldName, false); + mapped.remove(oldName); + mapped.put(newName, wasMapped); + } + } + return newSchema.stream().map(col -> { + Boolean isMapped = mapped.get(col.name()); + return new Column(col.name(), col.type(), col.originalTypes(), isMapped != null && isMapped); + }).toList(); + } + + /** + * Checks if the error is a full-text function/operator rejecting a field that is not from an index mapping. + * Uses the {@link Column#indexMapped()} flag from the current schema when available; falls back to + * command-history heuristics otherwise. + */ + static boolean isFieldFullTextError( + String errorMessage, + String query, + List previousCommands, + List currentSchema + ) { + String errorWithoutLineBreaks = normalizeErrorMessage(errorMessage); + Matcher m = NOT_A_FIELD_FROM_INDEX_PATTERN.matcher(errorWithoutLineBreaks); + if (m.matches() == false) { + return false; + } + String fieldName = unquote(m.group(1)); + + if (currentSchema != null && currentSchema.isEmpty() == false) { + for (Column col : currentSchema) { + if (col.name().equals(fieldName)) { + return col.indexMapped() == false; + } + } + // Field not found in schema — likely a pipeline artifact; allow the error + return true; + } return false; } + private static final Pattern FULL_TEXT_AFTER_SAMPLE_PATTERN = Pattern.compile( + ".*\\[(KQL|QSTR)] function cannot be used after SAMPLE.*", + Pattern.DOTALL + ); + /** * SAMPLE should not block QSTR/KQL when it appears after the WHERE containing them, but currently it does. * See https://github.com/elastic/elasticsearch/issues/142694 */ static boolean isFullTextAfterSampleBug(String errorMessage, String query) { return FULL_TEXT_AFTER_SAMPLE_PATTERN.matcher(normalizeErrorMessage(errorMessage)).matches() - && query.toLowerCase(java.util.Locale.ROOT).contains("| sample"); + && query.toLowerCase(Locale.ROOT).contains("| sample"); } - private static final Pattern FULL_TEXT_AFTER_SAMPLE_PATTERN = Pattern.compile( - ".*\\[(KQL|QSTR)] function cannot be used after SAMPLE.*", + private static final Pattern FULL_TEXT_AFTER_WHERE_PATTERN = Pattern.compile( + ".*(?:(?:\\[(?:KQL|QSTR|MATCH|MultiMatch|MatchPhrase)] function)|(?:\\[:\\] operator)) cannot be used after \\(?WHERE.*", Pattern.DOTALL ); @@ -603,9 +676,14 @@ static boolean isFullTextAfterWhereBugs(String errorMessage) { return FULL_TEXT_AFTER_WHERE_PATTERN.matcher(errorWithoutLineBreaks).matches(); } - private static final Pattern FULL_TEXT_AFTER_WHERE_PATTERN = Pattern.compile( - ".*(?:(?:\\[(?:KQL|QSTR|MATCH|MultiMatch|MatchPhrase)] function)|(?:\\[:\\] operator)) cannot be used after \\(?WHERE.*", - Pattern.DOTALL + private static final Pattern MULTI_MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bmulti_match\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" + ); + private static final Pattern MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bmatch\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" + ); + private static final Pattern QSTR_LENIENT_FALSE_PATTERN = Pattern.compile( + "(?i)\\bqstr\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" ); /** @@ -621,16 +699,6 @@ static boolean isLenientFalseFailedToCreateFullTextQueryError(String errorMessag || QSTR_LENIENT_FALSE_PATTERN.matcher(query).find(); } - private static final Pattern MULTI_MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bmulti_match\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - private static final Pattern MATCH_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bmatch\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - private static final Pattern QSTR_LENIENT_FALSE_PATTERN = Pattern.compile( - "(?i)\\bqstr\\s*\\([^)]*\\{[^}]*[\"']lenient[\"']\\s*:\\s*false[^}]*}[^)]*\\)" - ); - @Override @SuppressWarnings("unchecked") public QueryExecuted execute(String query, int depth) { @@ -659,9 +727,7 @@ private static List outputSchema(Map a) { if (cols == null) { return null; } - return cols.stream() - .map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))) - .collect(Collectors.toList()); + return cols.stream().map(x -> new Column((String) x.get(COLUMN_NAME), (String) x.get(COLUMN_TYPE), originalTypes(x))).toList(); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java index 8159c920ded77..ad2700a5f494d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/Column.java @@ -14,5 +14,17 @@ * @param name the field name * @param type the field type * @param originalTypes the original types, in case of a union type. + * @param indexMapped whether this column originates from an actual index mapping ({@code true}) or was + * computed/derived by a pipeline command such as EVAL, GROK, DISSECT, ENRICH, etc. ({@code false}). + * Full-text functions (match, match_phrase, multi_match, {@code :} operator) require index-mapped fields. + * An exception here is MV_EXPAND which, even if has as ouput a field from an index, this field is forbidden to be used + * in full-text functions, as well. See https://github.com/elastic/elasticsearch/issues/142713 */ -public record Column(String name, String type, List originalTypes) {} +public record Column(String name, String type, List originalTypes, boolean indexMapped) { + /** + * Backward-compatible constructor that defaults to {@code indexMapped = true}. + */ + public Column(String name, String type, List originalTypes) { + this(name, type, originalTypes, true); + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java index be3d547cf535c..89913a9cd2014 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/EnrichGenerator.java @@ -7,21 +7,28 @@ package org.elasticsearch.xpack.esql.generator.command.pipe; +import org.elasticsearch.xpack.esql.CsvTestsDataLoader; import org.elasticsearch.xpack.esql.generator.Column; import org.elasticsearch.xpack.esql.generator.EsqlQueryGenerator; import org.elasticsearch.xpack.esql.generator.QueryExecutor; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.elasticsearch.test.ESTestCase.randomFrom; public class EnrichGenerator implements CommandGenerator { public static final String ENRICH = "enrich"; + public static final String ENRICH_FIELDS = "enrichFields"; public static final CommandGenerator INSTANCE = new EnrichGenerator(); + private static final Pattern ENRICH_FIELDS_PATTERN = Pattern.compile("\"enrich_fields\"\\s*:\\s*\\[([^\\]]*)]"); + @Override public CommandDescription generate( List previousCommands, @@ -35,11 +42,28 @@ public CommandDescription generate( } // TODO add WITH - String cmdString = " | enrich " - + randomFrom(EsqlQueryGenerator.policiesOnKeyword(schema.enrichPolicies())).policyName() - + " on " - + field; - return new CommandDescription(ENRICH, this, cmdString, Map.of()); + CsvTestsDataLoader.EnrichConfig policy = randomFrom(EsqlQueryGenerator.policiesOnKeyword(schema.enrichPolicies())); + String cmdString = " | enrich " + policy.policyName() + " on " + field; + List enrichFields = parseEnrichFields(policy.loadPolicy()); + return new CommandDescription(ENRICH, this, cmdString, Map.of(ENRICH_FIELDS, enrichFields)); + } + + static List parseEnrichFields(String policyJson) { + Matcher m = ENRICH_FIELDS_PATTERN.matcher(policyJson); + if (m.find()) { + List fields = new ArrayList<>(); + for (String part : m.group(1).split(",")) { + String trimmed = part.trim(); + if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) { + trimmed = trimmed.substring(1, trimmed.length() - 1); + } + if (trimmed.isEmpty() == false) { + fields.add(trimmed); + } + } + return fields; + } + return List.of(); } @Override diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java index 536fbb3a06908..6e1105e735ddb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/source/FromGenerator.java @@ -29,13 +29,6 @@ public class FromGenerator implements CommandGenerator { */ public static final String UNMAPPED_FIELDS_ENABLED = "unmappedFieldsEnabled"; - /** - * Context key for the set of field names that come from the actual index mapping. - * Populated by the executor after the FROM command runs. - * Full-text functions that require FieldAttribute arguments use this to avoid computed columns. - */ - public static final String INDEX_FIELD_NAMES = "indexFieldNames"; - public static final String SET_UNMAPPED_FIELDS_PREFIX = "SET unmapped_fields=\"nullify\";"; /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java index 04bad2eecb127..83af796d01da9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/function/FullTextFunctionGenerator.java @@ -9,15 +9,11 @@ import org.elasticsearch.xpack.esql.generator.Column; import org.elasticsearch.xpack.esql.generator.command.CommandGenerator; -import org.elasticsearch.xpack.esql.generator.command.pipe.EvalGenerator; -import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.elasticsearch.test.ESTestCase.randomBoolean; @@ -68,48 +64,20 @@ private static boolean isQstrKqlAllowed(List indexFieldColumns(List columns, List previousCommands) { if (previousCommands == null || previousCommands.isEmpty()) { return null; } - Object stored = previousCommands.get(0).context().get(FromGenerator.INDEX_FIELD_NAMES); - if (stored instanceof Set == false) { + if (isFromSource(previousCommands.get(0)) == false) { return null; } - Set safeNames = new HashSet<>((Set) stored); - for (CommandGenerator.CommandDescription cmd : previousCommands) { - if ("eval".equals(cmd.commandName())) { - Object newCols = cmd.context().get(EvalGenerator.NEW_COLUMNS); - if (newCols instanceof List list) { - list.forEach(name -> safeNames.remove((String) name)); - } - } else if ("mv_expand".equals(cmd.commandName())) { - String expandedField = cmd.commandString().replaceFirst("(?i)^\\s*\\|\\s*mv_expand\\s+", "").trim(); - if (expandedField.startsWith("`") && expandedField.endsWith("`")) { - expandedField = expandedField.substring(1, expandedField.length() - 1); - } - safeNames.remove(expandedField); - } else if ("rename".equals(cmd.commandName())) { - String cmdStr = cmd.commandString().replaceFirst("(?i)^\\s*\\|\\s*rename\\s+", ""); - for (String pair : cmdStr.split(",")) { - Matcher m = RENAME_PAIR.matcher(pair); - if (m.matches()) { - String oldName = m.group(1); - String newName = m.group(2); - boolean wasSafe = safeNames.remove(oldName); - if (wasSafe) { - safeNames.add(newName); - } else { - safeNames.remove(newName); - } - } - } - } - } - return columns.stream().filter(c -> safeNames.contains(c.name())).toList(); + List result = columns.stream().filter(Column::indexMapped).toList(); + return result.isEmpty() ? null : result; } private static final Set MATCH_FIELD_TYPES = Set.of(