Skip to content

ESQL - Remove restrictions for disjunctions in full text functions #118544

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
carlosdelest merged 23 commits into from
Dec 18, 2024
Merged

ESQL - Remove restrictions for disjunctions in full text functions #118544

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
0766c5d
Remove restrictions for disjunctions in full text functions
carlosdelest Dec 12, 2024
ab91f4c
Fix CSV test
carlosdelest Dec 12, 2024
6097a0d
Update docs/changelog/118544.yaml
carlosdelest Dec 12, 2024
68e88ef
Merge remote-tracking branch 'carlosdelest/enhancement/esql-match-dis…
carlosdelest Dec 12, 2024
c198d91
Fixing tests
carlosdelest Dec 12, 2024
c91cf7c
Refactor detection method for full text functions
carlosdelest Dec 12, 2024
47be335
Add capabilities to tests
carlosdelest Dec 12, 2024
3a9e468
Fix test
carlosdelest Dec 13, 2024
da32a06
Fix mixed cluster test
carlosdelest Dec 13, 2024
60221ba
Check that all elements on each disjunction side have full text funct…
carlosdelest Dec 13, 2024
191e710
Add CSV tests
carlosdelest Dec 13, 2024
9860680
Merge branch 'main' into enhancement/esql-match-disjunction-restrictions
carlosdelest Dec 13, 2024
1056909
Fix merge
carlosdelest Dec 13, 2024
b3e9a64
Fix tests
carlosdelest Dec 13, 2024
657c447
Merge branch 'refs/heads/main' into enhancement/esql-match-disjunctio…
carlosdelest Dec 17, 2024
6449d22
Fix merge
carlosdelest Dec 17, 2024
8901591
Simplify disjunction algorithm
carlosdelest Dec 17, 2024
d213d76
Checkstyle
carlosdelest Dec 17, 2024
4968e9e
Fix test
carlosdelest Dec 17, 2024
317f7d8
Merge branch 'main' into enhancement/esql-match-disjunction-restrictions
carlosdelest Dec 17, 2024
1cd80aa
Merge branch 'main' into enhancement/esql-match-disjunction-restrictions
carlosdelest Dec 17, 2024
d4789dd
Fix test
carlosdelest Dec 17, 2024
d3b5dce
Spotless
carlosdelest Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/118544.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 118544
summary: ESQL - Remove restrictions for disjunctions in full text functions
area: ES|QL
type: enhancement
issues: []
1 change: 1 addition & 0 deletions x-pack/plugin/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
task.skipTest("privileges/11_builtin/Test get builtin privileges" ,"unnecessary to test compatibility")
task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors")
task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
})

74 changes: 74 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,80 @@ book_no:keyword | title:text
7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)
;

matchWithDisjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where match(author, "Vonnegut") or match(author, "Guinane")
| keep book_no, author;
ignoreOrder:true

book_no:keyword | author:text
2464 | Kurt Vonnegut
6970 | Edith Vonnegut
8956 | Kurt Vonnegut
3950 | Kurt Vonnegut
4382 | Carole Guinane
;

matchWithDisjunctionAndFiltersConjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where (match(author, "Vonnegut") or match(author, "Guinane")) and year > 1997
| keep book_no, author, year;
ignoreOrder:true

book_no:keyword | author:text | year:integer
6970 | Edith Vonnegut | 1998
4382 | Carole Guinane | 2001
;

matchWithDisjunctionAndConjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where (match(author, "Vonnegut") or match(author, "Marquez")) and match(description, "realism")
| keep book_no;

book_no:keyword
4814
;

matchWithMoreComplexDisjunctionAndConjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where (match(author, "Vonnegut") and match(description, "charming")) or (match(author, "Marquez") and match(description, "realism"))
| keep book_no;
ignoreOrder:true

book_no:keyword
6970
4814
;

matchWithDisjunctionIncludingConjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where match(author, "Vonnegut") or (match(author, "Marquez") and match(description, "realism"))
| keep book_no;
ignoreOrder:true

book_no:keyword
2464
6970
4814
8956
3950
;

Copy link
Member

@fang-xing-esql fang-xing-esql Dec 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a few more queries with nested AND/OR(in csv and yml tests), similar as the one below, and perhaps make it a bit more complicated? It will be helpful to catch some surprises.

curl -u elastic:password -v -X POST "localhost:9200/_query?format=txt&pretty" -H 'Content-Type: application/json' -d'
{
  "query": "from books | where (match(name, \"Space\") and length(name) > 0) or (match(author, \"Neal\") and length(author) > 0)"
}

  "error" : {
    "root_cause" : [
      {
        "type" : "ql_illegal_argument_exception",
        "reason" : "Unsupported expression [match(name, \"Space\")]"
      }
    ],
    "type" : "ql_illegal_argument_exception",
    "reason" : "Unsupported expression [match(name, \"Space\")]"
  },
  "status" : 500

Copy link
Contributor

@tteofili tteofili Dec 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

perhaps we can add an IT which programmatically adds a random number of nested disjunctions?

Copy link
Member Author

@carlosdelest carlosdelest Dec 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for checking this, I added some more tests as this test case needed some code changes.

perhaps we can add an IT which programmatically adds a random number of nested disjunctions?

Agreed, we need to change the tests for full text functions to include this kind of testing. Let's do that on a separate PR and issue.

matchWithFunctionPushedToLucene
required_capability: match_function

Expand Down
121 changes: 98 additions & 23 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,81 @@ book_no:keyword | title:text
7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1)
;


matchWithDisjunction
required_capability: match_operator_colon
required_capability: full_text_functions_disjunctions

from books
| where author : "Vonnegut" or author : "Guinane"
| keep book_no, author;
ignoreOrder:true

book_no:keyword | author:text
2464 | Kurt Vonnegut
6970 | Edith Vonnegut
8956 | Kurt Vonnegut
3950 | Kurt Vonnegut
4382 | Carole Guinane
;

matchWithDisjunctionAndFiltersConjunction
required_capability: match_operator_colon
required_capability: full_text_functions_disjunctions

from books
| where (author : "Vonnegut" or author : "Guinane") and year > 1997
| keep book_no, author, year;
ignoreOrder:true

book_no:keyword | author:text | year:integer
6970 | Edith Vonnegut | 1998
4382 | Carole Guinane | 2001
;

matchWithDisjunctionAndConjunction
required_capability: match_operator_colon
required_capability: full_text_functions_disjunctions

from books
| where (author : "Vonnegut" or author : "Marquez") and description : "realism"
| keep book_no;

book_no:keyword
4814
;

matchWithMoreComplexDisjunctionAndConjunction
required_capability: match_function
required_capability: full_text_functions_disjunctions

from books
| where (author : "Vonnegut" and description : "charming") or (author : "Marquez" and description : "realism")
| keep book_no;
ignoreOrder:true

book_no:keyword
6970
4814
;

matchWithDisjunctionIncludingConjunction
required_capability: match_operator_colon
required_capability: full_text_functions_disjunctions

from books
| where author : "Vonnegut" or (author : "Marquez" and description : "realism")
| keep book_no;
ignoreOrder:true

book_no:keyword
2464
6970
4814
8956
3950
;

matchWithFunctionPushedToLucene
required_capability: match_operator_colon

Expand Down Expand Up @@ -219,7 +294,7 @@ count(*): long | author.keyword:keyword
;

testMatchBooleanField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -235,7 +310,7 @@ Amabile | true | 2.09
;

testMatchIntegerField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -247,7 +322,7 @@ emp_no:integer | first_name:keyword
;

testMatchDoubleField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -259,7 +334,7 @@ emp_no:integer | salary_change:double
;

testMatchLongField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from date_nanos
Expand All @@ -271,7 +346,7 @@ num:long
;

testMatchUnsignedLongField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from ul_logs
Expand All @@ -283,7 +358,7 @@ bytes_out:unsigned_long
;

testMatchIpFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from sample_data
Expand All @@ -295,7 +370,7 @@ client_ip:ip | message:keyword
;

testMatchDateFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from date_nanos
Expand All @@ -307,7 +382,7 @@ millis:date
;

testMatchDateNanosFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from date_nanos
Expand All @@ -319,7 +394,7 @@ nanos:date_nanos
;

testMatchBooleanFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -335,7 +410,7 @@ Amabile | true | 2.09
;

testMatchIntegerFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -347,7 +422,7 @@ emp_no:integer | first_name:keyword
;

testMatchDoubleFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -359,7 +434,7 @@ emp_no:integer | salary_change:double
;

testMatchLongFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from date_nanos
Expand All @@ -371,7 +446,7 @@ num:long
;

testMatchUnsignedLongFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from ul_logs
Expand All @@ -383,7 +458,7 @@ bytes_out:unsigned_long
;

testMatchVersionFieldAsString
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from apps
Expand All @@ -395,7 +470,7 @@ bbbbb | 2.1
;

testMatchIntegerAsDouble
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -408,7 +483,7 @@ emp_no:integer | first_name:keyword
;

testMatchDoubleAsIntegerField
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees
Expand All @@ -423,7 +498,7 @@ emp_no:integer | height:double
;

testMatchMultipleFieldTypes
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -440,7 +515,7 @@ emp_as_int:integer | name_as_kw:keyword


testMatchMultipleFieldTypesKeywordText
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -455,7 +530,7 @@ Kazuhito
;

testMatchMultipleFieldTypesDoubleFloat
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -474,7 +549,7 @@ emp_no:integer | height_dbl:double
;

testMatchMultipleFieldTypesBooleanKeyword
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -491,7 +566,7 @@ true
;

testMatchMultipleFieldTypesLongUnsignedLong
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -506,7 +581,7 @@ avg_worked_seconds_ul:unsigned_long
;

testMatchMultipleFieldTypesDateNanosDate
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand All @@ -521,7 +596,7 @@ hire_date_nanos:date_nanos
;

testMatchWithWrongFieldValue
required_capability: match_function
required_capability: match_operator_colon
required_capability: match_additional_types

from employees,employees_incompatible
Expand Down
Loading