diff --git a/docs/changelog/118544.yaml b/docs/changelog/118544.yaml new file mode 100644 index 0000000000000..d59783c4e6194 --- /dev/null +++ b/docs/changelog/118544.yaml @@ -0,0 +1,5 @@ +pr: 118544 +summary: ESQL - Remove restrictions for disjunctions in full text functions +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index fb37fb3575551..aa6e8de4ec27c 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -94,5 +94,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("privileges/11_builtin/Test get builtin privileges" ,"unnecessary to test compatibility") task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors") task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") + task.skipTest("esql/180_match_operator/match with functions", "Error message changed") }) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec index 03b24555dbeff..5ea169e1b110d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -115,6 +115,80 @@ book_no:keyword | title:text 7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) ; +matchWithDisjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where match(author, "Vonnegut") or match(author, "Guinane") +| keep book_no, author; +ignoreOrder:true + +book_no:keyword | author:text +2464 | Kurt Vonnegut +6970 | Edith Vonnegut +8956 | Kurt Vonnegut +3950 | Kurt Vonnegut +4382 | Carole Guinane +; + +matchWithDisjunctionAndFiltersConjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where (match(author, "Vonnegut") or match(author, "Guinane")) and year > 1997 +| keep book_no, author, year; +ignoreOrder:true + +book_no:keyword | author:text | year:integer +6970 | Edith Vonnegut | 1998 +4382 | Carole Guinane | 2001 +; + +matchWithDisjunctionAndConjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where (match(author, "Vonnegut") or match(author, "Marquez")) and match(description, "realism") +| keep book_no; + +book_no:keyword +4814 +; + +matchWithMoreComplexDisjunctionAndConjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where (match(author, "Vonnegut") and match(description, "charming")) or (match(author, "Marquez") and match(description, "realism")) +| keep book_no; +ignoreOrder:true + +book_no:keyword +6970 +4814 +; + +matchWithDisjunctionIncludingConjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where match(author, "Vonnegut") or (match(author, "Marquez") and match(description, "realism")) +| keep book_no; +ignoreOrder:true + +book_no:keyword +2464 +6970 +4814 +8956 +3950 +; + matchWithFunctionPushedToLucene required_capability: match_function diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index 56f7f5ccd8823..7906f8b69162b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -102,6 +102,81 @@ book_no:keyword | title:text 7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) ; + +matchWithDisjunction +required_capability: match_operator_colon +required_capability: full_text_functions_disjunctions + +from books +| where author : "Vonnegut" or author : "Guinane" +| keep book_no, author; +ignoreOrder:true + +book_no:keyword | author:text +2464 | Kurt Vonnegut +6970 | Edith Vonnegut +8956 | Kurt Vonnegut +3950 | Kurt Vonnegut +4382 | Carole Guinane +; + +matchWithDisjunctionAndFiltersConjunction +required_capability: match_operator_colon +required_capability: full_text_functions_disjunctions + +from books +| where (author : "Vonnegut" or author : "Guinane") and year > 1997 +| keep book_no, author, year; +ignoreOrder:true + +book_no:keyword | author:text | year:integer +6970 | Edith Vonnegut | 1998 +4382 | Carole Guinane | 2001 +; + +matchWithDisjunctionAndConjunction +required_capability: match_operator_colon +required_capability: full_text_functions_disjunctions + +from books +| where (author : "Vonnegut" or author : "Marquez") and description : "realism" +| keep book_no; + +book_no:keyword +4814 +; + +matchWithMoreComplexDisjunctionAndConjunction +required_capability: match_function +required_capability: full_text_functions_disjunctions + +from books +| where (author : "Vonnegut" and description : "charming") or (author : "Marquez" and description : "realism") +| keep book_no; +ignoreOrder:true + +book_no:keyword +6970 +4814 +; + +matchWithDisjunctionIncludingConjunction +required_capability: match_operator_colon +required_capability: full_text_functions_disjunctions + +from books +| where author : "Vonnegut" or (author : "Marquez" and description : "realism") +| keep book_no; +ignoreOrder:true + +book_no:keyword +2464 +6970 +4814 +8956 +3950 +; + matchWithFunctionPushedToLucene required_capability: match_operator_colon @@ -219,7 +294,7 @@ count(*): long | author.keyword:keyword ; testMatchBooleanField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -235,7 +310,7 @@ Amabile | true | 2.09 ; testMatchIntegerField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -247,7 +322,7 @@ emp_no:integer | first_name:keyword ; testMatchDoubleField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -259,7 +334,7 @@ emp_no:integer | salary_change:double ; testMatchLongField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from date_nanos @@ -271,7 +346,7 @@ num:long ; testMatchUnsignedLongField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from ul_logs @@ -283,7 +358,7 @@ bytes_out:unsigned_long ; testMatchIpFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from sample_data @@ -295,7 +370,7 @@ client_ip:ip | message:keyword ; testMatchDateFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from date_nanos @@ -307,7 +382,7 @@ millis:date ; testMatchDateNanosFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from date_nanos @@ -319,7 +394,7 @@ nanos:date_nanos ; testMatchBooleanFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -335,7 +410,7 @@ Amabile | true | 2.09 ; testMatchIntegerFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -347,7 +422,7 @@ emp_no:integer | first_name:keyword ; testMatchDoubleFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -359,7 +434,7 @@ emp_no:integer | salary_change:double ; testMatchLongFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from date_nanos @@ -371,7 +446,7 @@ num:long ; testMatchUnsignedLongFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from ul_logs @@ -383,7 +458,7 @@ bytes_out:unsigned_long ; testMatchVersionFieldAsString -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from apps @@ -395,7 +470,7 @@ bbbbb | 2.1 ; testMatchIntegerAsDouble -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -408,7 +483,7 @@ emp_no:integer | first_name:keyword ; testMatchDoubleAsIntegerField -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees @@ -423,7 +498,7 @@ emp_no:integer | height:double ; testMatchMultipleFieldTypes -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -440,7 +515,7 @@ emp_as_int:integer | name_as_kw:keyword testMatchMultipleFieldTypesKeywordText -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -455,7 +530,7 @@ Kazuhito ; testMatchMultipleFieldTypesDoubleFloat -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -474,7 +549,7 @@ emp_no:integer | height_dbl:double ; testMatchMultipleFieldTypesBooleanKeyword -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -491,7 +566,7 @@ true ; testMatchMultipleFieldTypesLongUnsignedLong -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -506,7 +581,7 @@ avg_worked_seconds_ul:unsigned_long ; testMatchMultipleFieldTypesDateNanosDate -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible @@ -521,7 +596,7 @@ hire_date_nanos:date_nanos ; testMatchWithWrongFieldValue -required_capability: match_function +required_capability: match_operator_colon required_capability: match_additional_types from employees,employees_incompatible diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java index 58b1652653ca3..ad90bbf6ae9db 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java @@ -168,7 +168,7 @@ public void testWhereMatchWithScoringNoSort() { var query = """ FROM test METADATA _score - | WHERE content:"fox" + | WHERE match(content, "fox") | KEEP id, _score """; @@ -182,7 +182,7 @@ public void testWhereMatchWithScoringNoSort() { public void testNonExistingColumn() { var query = """ FROM test - | WHERE something:"fox" + | WHERE match(something, "fox") """; var error = expectThrows(VerificationException.class, () -> run(query)); @@ -193,14 +193,14 @@ public void testWhereMatchEvalColumn() { var query = """ FROM test | EVAL upper_content = to_upper(content) - | WHERE upper_content:"FOX" + | WHERE match(upper_content, "FOX") | KEEP id """; var error = expectThrows(VerificationException.class, () -> run(query)); assertThat( error.getMessage(), - containsString("[:] operator cannot operate on [upper_content], which is not a field from an index mapping") + containsString("[MATCH] function cannot operate on [upper_content], which is not a field from an index mapping") ); } @@ -209,13 +209,13 @@ public void testWhereMatchOverWrittenColumn() { FROM test | DROP content | EVAL content = CONCAT("document with ID ", to_str(id)) - | WHERE content:"document" + | WHERE match(content, "document") """; var error = expectThrows(VerificationException.class, () -> run(query)); assertThat( error.getMessage(), - containsString("[:] operator cannot operate on [content], which is not a field from an index mapping") + containsString("[MATCH] function cannot operate on [content], which is not a field from an index mapping") ); } @@ -223,7 +223,7 @@ public void testWhereMatchAfterStats() { var query = """ FROM test | STATS count(*) - | WHERE content:"fox" + | WHERE match(content, "fox") """; var error = expectThrows(VerificationException.class, () -> run(query)); @@ -233,14 +233,15 @@ public void testWhereMatchAfterStats() { public void testWhereMatchWithFunctions() { var query = """ FROM test - | WHERE content:"fox" OR to_upper(content) == "FOX" + | WHERE match(content, "fox") OR to_upper(content) == "FOX" """; var error = expectThrows(ElasticsearchException.class, () -> run(query)); assertThat( error.getMessage(), containsString( - "Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. " - + "[:] operator can't be used as part of an or condition" + "Invalid condition [match(content, \"fox\") OR to_upper(content) == \"FOX\"]. " + + "Full text functions can be used in an OR condition," + + " but only if just full text functions are used in the OR condition" ) ); } @@ -248,24 +249,24 @@ public void testWhereMatchWithFunctions() { public void testWhereMatchWithRow() { var query = """ ROW content = "a brown fox" - | WHERE content:"fox" + | WHERE match(content, "fox") """; var error = expectThrows(ElasticsearchException.class, () -> run(query)); assertThat( error.getMessage(), - containsString("[:] operator cannot operate on [\"a brown fox\"], which is not a field from an index mapping") + containsString("[MATCH] function cannot operate on [\"a brown fox\"], which is not a field from an index mapping") ); } public void testMatchWithinEval() { var query = """ FROM test - | EVAL matches_query = content:"fox" + | EVAL matches_query = match(content, "fox") """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE commands")); } private void createAndPopulateIndex() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index d0a641f086fe4..758878b46d51f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -216,7 +216,8 @@ public void testWhereMatchWithFunctions() { error.getMessage(), containsString( "Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. " - + "[:] operator can't be used as part of an or condition" + + "Full text functions can be used in an OR condition, " + + "but only if just full text functions are used in the OR condition" ) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4fcabb02b2d4f..39afd91f347d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -582,7 +582,12 @@ public enum Cap { /** * Fix for regex folding with case-insensitive pattern https://github.com/elastic/elasticsearch/issues/118371 */ - FIXED_REGEX_FOLD; + FIXED_REGEX_FOLD, + + /** + * Full text functions can be used in disjunctions + */ + FULL_TEXT_FUNCTIONS_DISJUNCTIONS; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index f01cc265e330b..6b98b7d69834f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -766,41 +766,78 @@ private static void checkRemoteEnrich(LogicalPlan plan, Set failures) { } /** - * Checks whether a condition contains a disjunction with the specified typeToken. Adds to failure if it does. + * Checks whether a condition contains a disjunction with a full text search. + * If it does, check that every element of the disjunction is a full text search or combinations (AND, OR, NOT) of them. + * If not, add a failure to the failures collection. * - * @param condition condition to check for disjunctions + * @param condition condition to check for disjunctions of full text searches * @param typeNameProvider provider for the type name to add in the failure message * @param failures failures collection to add to */ - private static void checkNotPresentInDisjunctions( + private static void checkFullTextSearchDisjunctions( Expression condition, java.util.function.Function typeNameProvider, Set failures ) { - condition.forEachUp(Or.class, or -> { - checkNotPresentInDisjunctions(or.left(), or, typeNameProvider, failures); - checkNotPresentInDisjunctions(or.right(), or, typeNameProvider, failures); + int failuresCount = failures.size(); + condition.forEachDown(Or.class, or -> { + if (failures.size() > failuresCount) { + // Exit early if we already have a failures + return; + } + boolean hasFullText = or.anyMatch(FullTextFunction.class::isInstance); + if (hasFullText) { + boolean hasOnlyFullText = onlyFullTextFunctionsInExpression(or); + if (hasOnlyFullText == false) { + failures.add( + fail( + or, + "Invalid condition [{}]. Full text functions can be used in an OR condition, " + + "but only if just full text functions are used in the OR condition", + or.sourceText() + ) + ); + } + } }); } /** - * Checks whether a condition contains a disjunction with the specified typeToken. Adds to failure if it does. + * Checks whether an expression contains just full text functions or negations (NOT) and combinations (AND, OR) of full text functions * - * @param parentExpression parent expression to add to the failure message - * @param or disjunction that is being checked - * @param failures failures collection to add to + * @param expression expression to check + * @return true if all children are full text functions or negations of full text functions, false otherwise */ - private static void checkNotPresentInDisjunctions( - Expression parentExpression, - Or or, - java.util.function.Function elementName, - Set failures - ) { - parentExpression.forEachDown(FullTextFunction.class, ftp -> { - failures.add( - fail(or, "Invalid condition [{}]. {} can't be used as part of an or condition", or.sourceText(), elementName.apply(ftp)) - ); - }); + private static boolean onlyFullTextFunctionsInExpression(Expression expression) { + if (expression instanceof FullTextFunction) { + return true; + } else if (expression instanceof Not) { + return onlyFullTextFunctionsInExpression(expression.children().get(0)); + } else if (expression instanceof BinaryLogic binaryLogic) { + return onlyFullTextFunctionsInExpression(binaryLogic.left()) && onlyFullTextFunctionsInExpression(binaryLogic.right()); + } + + return false; + } + + /** + * Checks whether an expression contains a full text function as part of it + * + * @param expression expression to check + * @return true if the expression or any of its children is a full text function, false otherwise + */ + private static boolean anyFullTextFunctionsInExpression(Expression expression) { + if (expression instanceof FullTextFunction) { + return true; + } + + for (Expression child : expression.children()) { + if (anyFullTextFunctionsInExpression(child)) { + return true; + } + } + + return false; } /** @@ -870,7 +907,7 @@ private static void checkFullTextQueryFunctions(LogicalPlan plan, Set f m -> "[" + m.functionName() + "] " + m.functionType(), failures ); - checkNotPresentInDisjunctions(condition, ftf -> "[" + ftf.functionName() + "] " + ftf.functionType(), failures); + checkFullTextSearchDisjunctions(condition, ftf -> "[" + ftf.functionName() + "] " + ftf.functionType(), failures); checkFullTextFunctionsParents(condition, failures); } else { plan.forEachExpression(FullTextFunction.class, ftf -> { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 182e87d1ab9dd..a1e29117a25d3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1166,12 +1166,14 @@ public void testMatchInsideEval() throws Exception { public void testMatchFilter() throws Exception { assertEquals( "1:19: Invalid condition [first_name:\"Anna\" or starts_with(first_name, \"Anne\")]. " - + "[:] operator can't be used as part of an or condition", + + "Full text functions can be used in an OR condition, " + + "but only if just full text functions are used in the OR condition", error("from test | where first_name:\"Anna\" or starts_with(first_name, \"Anne\")") ); assertEquals( - "1:51: Invalid condition [first_name:\"Anna\" OR new_salary > 100]. " + "[:] operator can't be used as part of an or condition", + "1:51: Invalid condition [first_name:\"Anna\" OR new_salary > 100]. Full text functions can be" + + " used in an OR condition, but only if just full text functions are used in the OR condition", error("from test | eval new_salary = salary + 10 | where first_name:\"Anna\" OR new_salary > 100") ); } @@ -1409,48 +1411,56 @@ public void testMatchOperatorWithDisjunctions() { } private void checkWithDisjunctions(String functionName, String functionInvocation, String functionType) { + String expression = functionInvocation + " or length(first_name) > 12"; + checkdisjunctionError("1:19", expression, functionName, functionType); + expression = "(" + functionInvocation + " or first_name is not null) or (length(first_name) > 12 and match(last_name, \"Smith\"))"; + checkdisjunctionError("1:19", expression, functionName, functionType); + expression = functionInvocation + " or (last_name is not null and first_name is null)"; + checkdisjunctionError("1:19", expression, functionName, functionType); + } + + private void checkdisjunctionError(String position, String expression, String functionName, String functionType) { assertEquals( LoggerMessageFormat.format( null, - "1:19: Invalid condition [{} or length(first_name) > 12]. " - + "[{}] " - + functionType - + " can't be used as part of an or condition", - functionInvocation, - functionName - ), - error("from test | where " + functionInvocation + " or length(first_name) > 12") - ); - assertEquals( - LoggerMessageFormat.format( - null, - "1:19: Invalid condition [({} and first_name is not null) or (length(first_name) > 12 and first_name is null)]. " - + "[{}] " - + functionType - + " can't be used as part of an or condition", - functionInvocation, - functionName - ), - error( - "from test | where (" - + functionInvocation - + " and first_name is not null) or (length(first_name) > 12 and first_name is null)" - ) - ); - assertEquals( - LoggerMessageFormat.format( - null, - "1:19: Invalid condition [({} and first_name is not null) or first_name is null]. " - + "[{}] " - + functionType - + " can't be used as part of an or condition", - functionInvocation, - functionName + "{}: Invalid condition [{}]. Full text functions can be used in an OR condition, " + + "but only if just full text functions are used in the OR condition", + position, + expression ), - error("from test | where (" + functionInvocation + " and first_name is not null) or first_name is null") + error("from test | where " + expression) ); } + public void testFullTextFunctionsDisjunctions() { + checkWithFullTextFunctionsDisjunctions("MATCH", "match(last_name, \"Smith\")", "function"); + checkWithFullTextFunctionsDisjunctions(":", "last_name : \"Smith\"", "operator"); + checkWithFullTextFunctionsDisjunctions("QSTR", "qstr(\"last_name: Smith\")", "function"); + + assumeTrue("KQL function capability not available", EsqlCapabilities.Cap.KQL_FUNCTION.isEnabled()); + checkWithFullTextFunctionsDisjunctions("KQL", "kql(\"last_name: Smith\")", "function"); + } + + private void checkWithFullTextFunctionsDisjunctions(String functionName, String functionInvocation, String functionType) { + + String expression = functionInvocation + " or length(first_name) > 10"; + checkdisjunctionError("1:19", expression, functionName, functionType); + + expression = "match(last_name, \"Anneke\") or (" + functionInvocation + " and length(first_name) > 10)"; + checkdisjunctionError("1:19", expression, functionName, functionType); + + expression = "(" + + functionInvocation + + " and length(first_name) > 0) or (match(last_name, \"Anneke\") and length(first_name) > 10)"; + checkdisjunctionError("1:19", expression, functionName, functionType); + + query("from test | where " + functionInvocation + " or match(first_name, \"Anna\")"); + query("from test | where " + functionInvocation + " or not match(first_name, \"Anna\")"); + query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and length(first_name) > 10"); + query("from test | where (" + functionInvocation + " or match(first_name, \"Anna\")) and match(last_name, \"Smith\")"); + query("from test | where " + functionInvocation + " or (match(first_name, \"Anna\") and match(last_name, \"Smith\"))"); + } + public void testQueryStringFunctionWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"first_name: Anna\")", "function"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 879a413615202..406e27c1517e5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.license.XPackLicenseState; @@ -57,6 +58,7 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; +import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -1543,6 +1545,46 @@ public void testMultipleMatchFilterPushdown() { assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); } + public void testFullTextFunctionsDisjunctionPushdown() { + String query = """ + from test + | where (match(first_name, "Anna") or qstr("first_name: Anneke")) and last_name: "Smith" + | sort emp_no + """; + var plan = plannerOptimizer.plan(query); + var topNExec = as(plan, TopNExec.class); + var exchange = as(topNExec.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query(); + var expectedLuceneQuery = new BoolQueryBuilder().must( + new BoolQueryBuilder().should(new MatchQueryBuilder("first_name", "Anna").lenient(true)) + .should(new QueryStringQueryBuilder("first_name: Anneke")) + ).must(new MatchQueryBuilder("last_name", "Smith").lenient(true)); + assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); + } + + public void testFullTextFunctionsDisjunctionWithFiltersPushdown() { + String query = """ + from test + | where (first_name:"Anna" or first_name:"Anneke") and length(last_name) > 5 + | sort emp_no + """; + var plan = plannerOptimizer.plan(query); + var topNExec = as(plan, TopNExec.class); + var exchange = as(topNExec.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + var secondTopNExec = as(fieldExtract.child(), TopNExec.class); + var secondFieldExtract = as(secondTopNExec.child(), FieldExtractExec.class); + var filterExec = as(secondFieldExtract.child(), FilterExec.class); + var thirdFilterExtract = as(filterExec.child(), FieldExtractExec.class); + var actualLuceneQuery = as(thirdFilterExtract.child(), EsQueryExec.class).query(); + var expectedLuceneQuery = new BoolQueryBuilder().should(new MatchQueryBuilder("first_name", "Anna").lenient(true)) + .should(new MatchQueryBuilder("first_name", "Anneke").lenient(true)); + assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); + } + /** * Expecting * LimitExec[1000[INTEGER]] diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml index 663c0dc78acb3..118783b412d48 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml @@ -170,7 +170,7 @@ setup: - match: { error.reason: "Found 1 problem\nline 1:36: Unknown column [content], did you mean [count(*)]?" } --- -"match with functions": +"match with disjunctions": - do: catch: bad_request allowed_warnings_regex: @@ -181,7 +181,20 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:19: Invalid condition [content:\"fox\" OR to_upper(content) == \"FOX\"]. [:] operator can't be used as part of an or condition" } + - match: { error.reason: "/.+Invalid\\ condition\\ \\[content\\:\"fox\"\\ OR\\ to_upper\\(content\\)\\ ==\\ \"FOX\"\\]\\./" } + + - do: + catch: bad_request + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | WHERE content:"fox" OR to_upper(content) == "FOX"' + + - match: { status: 400 } + - match: { error.type: verification_exception } + - match: { error.reason: "/.+Invalid\\ condition\\ \\[content\\:\"fox\"\\ OR\\ to_upper\\(content\\)\\ ==\\ \"FOX\"\\]\\./" } + --- "match within eval":