diff --git a/docs/changelog/137024.yaml b/docs/changelog/137024.yaml new file mode 100644 index 0000000000000..6f4ee11249d9a --- /dev/null +++ b/docs/changelog/137024.yaml @@ -0,0 +1,5 @@ +pr: 137024 +summary: Catch-and-rethrow `TooComplexToDeterminizeException` within ESQL +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java index cd8f0b75839c8..bce03e8d9dabb 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java @@ -11,12 +11,21 @@ import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; public abstract class AbstractStringPattern implements StringPattern { private Automaton automaton; - public abstract Automaton createAutomaton(boolean ignoreCase); + public final Automaton createAutomaton(boolean ignoreCase) { + try { + return doCreateAutomaton(ignoreCase); + } catch (TooComplexToDeterminizeException e) { + throw new IllegalArgumentException("Pattern was too complex to determinize", e); + } + } + + protected abstract Automaton doCreateAutomaton(boolean ignoreCase); private Automaton automaton() { if (automaton == null) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java index 72b8c2efb2eba..732127816c452 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java @@ -34,7 +34,7 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Automaton createAutomaton(boolean ignoreCase) { + protected Automaton doCreateAutomaton(boolean ignoreCase) { int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0; return Operations.determinize( new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(), diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java index be62d189bafa4..34642deeed58f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java @@ -43,7 +43,7 @@ public List patternList() { * We create a single automaton that is the union of all individual automatons to improve performance */ @Override - public Automaton createAutomaton(boolean ignoreCase) { + protected Automaton doCreateAutomaton(boolean ignoreCase) { List automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList(); Automaton result = Operations.union(automatonList); return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java index 35c5674d08a87..49583c33af263 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java @@ -53,7 +53,7 @@ public String pattern() { } @Override - public Automaton createAutomaton(boolean ignoreCase) { + protected Automaton doCreateAutomaton(boolean ignoreCase) { return ignoreCase ? Operations.determinize( new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java index 64857469c6032..51c81e62c7e82 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java @@ -51,7 +51,7 @@ public List patternList() { * We create a single automaton that is the union of all individual automata to improve performance */ @Override - public Automaton createAutomaton(boolean ignoreCase) { + protected Automaton doCreateAutomaton(boolean ignoreCase) { List automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList(); Automaton result = Operations.union(automatonList); return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java index f0abdb3ecbcf0..e40e792c70a90 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java @@ -106,4 +106,12 @@ public void testRegexExactMatchWithEmptyMatch() { assertNotNull(exactMatchRLike("#|foo")); assertNotNull(exactMatchRLike("foo|#")); } + + public void testTooComplexPattern() { + var e = expectThrows(IllegalArgumentException.class, () -> rlike("(a|b)*a(a|b){13}").createAutomaton(false)); + assertEquals("Pattern was too complex to determinize", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> like("*a?????????????").createAutomaton(false)); + assertEquals("Pattern was too complex to determinize", e.getMessage()); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 29a8101dd9324..0a7e787ca9367 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1579,6 +1579,11 @@ public enum Cap { */ FORBID_LIMIT_BEFORE_INLINE_STATS(INLINE_STATS.enabled), + /** + * Catch-and-rethrow determinization complexity errors as 400s rather than 500s + */ + HANDLE_DETERMINIZATION_COMPLEXITY, + /** * Support for the TRANGE function */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java index 4469f86dc7469..806ca4c78ba84 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java @@ -11,6 +11,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.apache.lucene.util.automaton.Transition; import org.apache.lucene.util.automaton.UTF32ToUTF8; import org.elasticsearch.compute.ann.Evaluator; @@ -36,7 +37,13 @@ public static EvalOperator.ExpressionEvaluator.Factory toEvaluator( * ByteRunAutomaton has a way to convert utf32 to utf8, but if we used it * we couldn’t get a nice toDot - so we call UTF32ToUTF8 ourselves. */ - Automaton automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + Automaton automaton; + try { + automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + } catch (TooComplexToDeterminizeException e) { + throw new IllegalArgumentException("Pattern was too complex to determinize", e); + } + ByteRunAutomaton run = new ByteRunAutomaton(automaton, true); return new AutomataMatchEvaluator.Factory(source, field, run, toDot(automaton)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index f11473dabc21d..353029d651b05 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -15,6 +15,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.regex.Regex; @@ -485,12 +486,18 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat list.add(o instanceof Automaton a ? a : Automata.makeString(o.toString())); } // use the fast run variant - result = new UnresolvedNamePattern( - src, - new CharacterRunAutomaton(Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)), - patternString.toString(), - nameString.toString() - ); + try { + result = new UnresolvedNamePattern( + src, + new CharacterRunAutomaton( + Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT) + ), + patternString.toString(), + nameString.toString() + ); + } catch (TooComplexToDeterminizeException e) { + throw new ParsingException("Pattern was too complex to determinize", e); + } if (qualifier != null) { throw qualifiersUnsupportedInPatterns(src, qualifiedCtx.getText()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index a150e1270ac9d..eec71ffbd3152 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -1368,6 +1368,13 @@ public void testLikeRLike() { ); } + public void testIdentifierPatternTooComplex() { + // It is incredibly unlikely that we will see this limit hit in practice + // The repetition value 2450 was a ballpark estimate and validated experimentally + String explodingWildcard = "a*".repeat(2450); + expectError("FROM a | KEEP " + explodingWildcard, "Pattern was too complex to determinize"); + } + public void testEnrich() { assertEqualsIgnoringIds( new Enrich( diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index bfb49bdc7e5c5..b4bfd99dca5fe 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -616,3 +616,32 @@ setup: esql.query: body: query: 'FROM test | EVAL tag = name::text | KEEP name' +--- +"like with overly complex pattern": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ handle_determinization_complexity ] + reason: "determinization errors not yet rethrown" + - do: + catch: /Pattern was too complex to determinize/ + esql.query: + body: + query: 'FROM test | WHERE job LIKE "*a?????????????"' + +--- +"rlike with overly complex pattern": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ handle_determinization_complexity ] + reason: "determinization errors not yet rethrown" + - do: + catch: /Pattern was too complex to determinize/ + esql.query: + body: + query: 'FROM test | WHERE job RLIKE "(a|b)*a(a|b){13}"'