Skip to content

[ESQL] Catch-and-rethrow TooComplexToDeterminizeException #137024

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MattAlp merged 9 commits into from
Oct 26, 2025
Merged

[ESQL] Catch-and-rethrow TooComplexToDeterminizeException #137024

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
938d429
Remove staged test changes
MattAlp Oct 22, 2025
22b666a
Add fundamental tests
MattAlp Oct 23, 2025
a8def74
Update docs/changelog/137024.yaml
MattAlp Oct 23, 2025
ad5d22e
Spotless re-run
MattAlp Oct 23, 2025
0f6eb60
Merge branch 'main' into esql-catch-determinization-errors
MattAlp Oct 23, 2025
61e29c8
Add ESQL like & rlike complexity YAML tests
MattAlp Oct 24, 2025
7ea2d7b
Add determinization error handling ESQL capability
MattAlp Oct 24, 2025
e616071
Merge branch 'main' into esql-catch-determinization-errors
MattAlp Oct 24, 2025
74458bb
[CI] Auto commit changes from spotless
Oct 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/137024.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 137024
summary: Catch-and-rethrow `TooComplexToDeterminizeException` within ESQL
area: ES|QL
type: bug
issues: []
11 changes: 10 additions & 1 deletion ...a/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,21 @@
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;

public abstract class AbstractStringPattern implements StringPattern {

private Automaton automaton;

public abstract Automaton createAutomaton(boolean ignoreCase);
public final Automaton createAutomaton(boolean ignoreCase) {
try {
return doCreateAutomaton(ignoreCase);
} catch (TooComplexToDeterminizeException e) {
throw new IllegalArgumentException("Pattern was too complex to determinize", e);
}
}

protected abstract Automaton doCreateAutomaton(boolean ignoreCase);

private Automaton automaton() {
if (automaton == null) {
Expand Down
2 changes: 1 addition & 1 deletion .../main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public void writeTo(StreamOutput out) throws IOException {
}

@Override
public Automaton createAutomaton(boolean ignoreCase) {
protected Automaton doCreateAutomaton(boolean ignoreCase) {
int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0;
return Operations.determinize(
new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(),
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public List<RLikePattern> patternList() {
* We create a single automaton that is the union of all individual automatons to improve performance
*/
@Override
public Automaton createAutomaton(boolean ignoreCase) {
protected Automaton doCreateAutomaton(boolean ignoreCase) {
List<Automaton> automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList();
Automaton result = Operations.union(automatonList);
return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
Expand Down
2 changes: 1 addition & 1 deletion ...in/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public String pattern() {
}

@Override
public Automaton createAutomaton(boolean ignoreCase) {
protected Automaton doCreateAutomaton(boolean ignoreCase) {
return ignoreCase
? Operations.determinize(
new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE)
Expand Down
2 changes: 1 addition & 1 deletion ...ava/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public List<WildcardPattern> patternList() {
* We create a single automaton that is the union of all individual automata to improve performance
*/
@Override
public Automaton createAutomaton(boolean ignoreCase) {
protected Automaton doCreateAutomaton(boolean ignoreCase) {
List<Automaton> automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList();
Automaton result = Operations.union(automatonList);
return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
Expand Down
8 changes: 8 additions & 0 deletions ...java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,12 @@ public void testRegexExactMatchWithEmptyMatch() {
assertNotNull(exactMatchRLike("#|foo"));
assertNotNull(exactMatchRLike("foo|#"));
}

public void testTooComplexPattern() {
var e = expectThrows(IllegalArgumentException.class, () -> rlike("(a|b)*a(a|b){13}").createAutomaton(false));
assertEquals("Pattern was too complex to determinize", e.getMessage());

e = expectThrows(IllegalArgumentException.class, () -> like("*a?????????????").createAutomaton(false));
assertEquals("Pattern was too complex to determinize", e.getMessage());
}
}
9 changes: 8 additions & 1 deletion ...in/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.automaton.UTF32ToUTF8;
import org.elasticsearch.compute.ann.Evaluator;
Expand All @@ -36,7 +37,13 @@ public static EvalOperator.ExpressionEvaluator.Factory toEvaluator(
* ByteRunAutomaton has a way to convert utf32 to utf8, but if we used it
* we couldn’t get a nice toDot - so we call UTF32ToUTF8 ourselves.
*/
Automaton automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
Automaton automaton;
try {
automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
} catch (TooComplexToDeterminizeException e) {
throw new IllegalArgumentException("Pattern was too complex to determinize", e);
}

ByteRunAutomaton run = new ByteRunAutomaton(automaton, true);
return new AutomataMatchEvaluator.Factory(source, field, run, toDot(automaton));
}
Expand Down
19 changes: 13 additions & 6 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.regex.Regex;
Expand Down Expand Up @@ -485,12 +486,18 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat
list.add(o instanceof Automaton a ? a : Automata.makeString(o.toString()));
}
// use the fast run variant
result = new UnresolvedNamePattern(
src,
new CharacterRunAutomaton(Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)),
patternString.toString(),
nameString.toString()
);
try {
result = new UnresolvedNamePattern(
src,
new CharacterRunAutomaton(
Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
),
patternString.toString(),
nameString.toString()
);
} catch (TooComplexToDeterminizeException e) {
throw new ParsingException("Pattern was too complex to determinize", e);
}

if (qualifier != null) {
throw qualifiersUnsupportedInPatterns(src, qualifiedCtx.getText());
Expand Down
7 changes: 7 additions & 0 deletions ...k/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1333,6 +1333,13 @@ public void testLikeRLike() {
);
}

public void testIdentifierPatternTooComplex() {
// It is incredibly unlikely that we will see this limit hit in practice

@MattAlp MattAlp Oct 23, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I question if we really want to test for this, but it doesn't hurt to leave it in.

// The repetition value 2450 was a ballpark estimate and validated experimentally
String explodingWildcard = "a*".repeat(2450);
expectError("FROM a | KEEP " + explodingWildcard, "Pattern was too complex to determinize");
}

public void testEnrich() {
assertEquals(
new Enrich(
Expand Down