Additional track changes for ES|QL full text functions

elastic/logs/challenges/logging-querying-esql.json

@@ -80,70 +80,98 @@
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["esql", "search"]
+      "tags": ["esql", "full-text"]
     },
     {
-      "operation": "kafka_match_esql_query",
+      "operation": "apache_equals_esql_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["esql", "search"]
+      "tags": ["esql", "like"]
+    },
+    {
+      "operation": "kafka_qstr_esql_query",
+      "clients": {{ p_search_clients }},
+      "warmup-iterations": {{ warmup_iterations | default(20) }},
+      "iterations": {{ iterations | default(100) }},
+      "tags": ["esql", "full-text"]
+    },
+    {
+      "operation": "kafka_like_esql_query",
+      "clients": {{ p_search_clients }},
+      "warmup-iterations": {{ warmup_iterations | default(20) }},
+      "iterations": {{ iterations | default(100) }},
+      "tags": ["esql", "like"]
     },
     {
       "operation": "qstr_esql_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["esql", "search"]
+      "tags": ["esql", "full-text"]
     },
     {
       "operation": "kql_esql_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["esql", "search"]
+      "tags": ["esql", "full-text"]
+    },
+    {
+      "operation": "kql_like_esql_query",
+      "clients": {{ p_search_clients }},
+      "warmup-iterations": {{ warmup_iterations | default(20) }},
+      "iterations": {{ iterations | default(100) }},
+      "tags": ["esql", "like"]
     },
     {
       "operation": "syslog_qstr_esql_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["esql", "search"]
+      "tags": ["esql", "full-text"]
+    },
+    {
+      "operation": "syslog_like_esql_query",
+      "clients": {{ p_search_clients }},
+      "warmup-iterations": {{ warmup_iterations | default(20) }},
+      "iterations": {{ iterations | default(100) }},
+      "tags": ["esql", "like"]
     }{%- if build_flavor != "serverless" or serverless_operator == true %},
     {
       "operation": "apache_match_query_dsl_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["search"]
+      "tags": ["search", "full-text"]
     },
     {
-      "operation": "kafka_query_dsl_query",
+      "operation": "kafka_qstr_dsl_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["search"]
+      "tags": ["search", "full-text"]
     },
     {
       "operation": "qstr_query_dsl_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["search"]
+      "tags": ["search", "full-text"]
     },
     {
       "operation": "kql_query_dsl_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["search"]
+      "tags": ["search", "full-text"]
     },
     {
       "operation": "syslog_query_dsl_query",
       "clients": {{ p_search_clients }},
       "warmup-iterations": {{ warmup_iterations | default(20) }},
       "iterations": {{ iterations | default(100) }},
-      "tags": ["search"]
+      "tags": ["search", "full-text"]
     },
     {
       "operation": "enable_query_cache",

elastic/logs/operations/esql-full-text-search.json

@@ -2,7 +2,30 @@
     {
       "name": "apache_match_esql_query",
       "operation-type": "esql",
-      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset: \"apache.access\" AND http.response.status_code: 404 AND user_agent.name: \"Firefox\" | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | SORT @timestamp DESC | LIMIT 20"
+      "query": "FROM {{p_esql_target_prefix}}logs-*| WHERE event.dataset: \"apache.access\" AND http.response.status_code: 404 AND user_agent.name: \"Firefox\" | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T09:10:05.477Z",
+            "lte": "{{bulk_end_date}}T21:10:05.477Z"
+          }
+        }
+      }
+    },
+    {
+      "name": "apache_equals_esql_query",
+      "operation-type": "esql",
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset == \"apache.access\" AND http.response.status_code == 404 AND user_agent.name == \"Firefox\" | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T09:10:05.477Z",
+            "lte": "{{bulk_end_date}}T21:10:05.477Z"
+          }
+        }
+      }
     },
     {
       "name": "apache_match_query_dsl_query",
@@ -48,16 +71,47 @@
             "should": [],
             "must_not": []
           }
+        },
+        "size": 500,
+        "sort": [
+          {
+            "@timestamp": {
+              "order": "desc"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "name": "kafka_qstr_esql_query",
+      "operation-type": "esql",
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE QSTR(\"event.dataset: kafka.log AND Connection * disconnected\") | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T09:10:05.477Z",
+            "lte": "{{bulk_end_date}}T21:52:08.615Z"
+          }
         }
       }
     },
     {
-      "name": "kafka_match_esql_query",
+      "name": "kafka_like_esql_query",
       "operation-type": "esql",
-      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset: \"kafka.log\" AND QSTR(\"Connection * disconnected\") | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | SORT @timestamp DESC | LIMIT 500"
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset == \"kafka.log\" AND message LIKE \"Connection * disconnected\" | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T09:10:05.477Z",
+            "lte": "{{bulk_end_date}}T21:52:08.615Z"
+          }
+        }
+      }
     },
     {
-      "name": "kafka_query_dsl_query",
+      "name": "kafka_qstr_dsl_query",
       "operation-type": "search",
       "index": "{{p_esql_target_prefix}}logs-*",
       "body": {
@@ -69,8 +123,8 @@
                 "bool": {
                   "should": [
                     {
-                      "exists": {
-                        "field": "kafka.log.trace.class"
+                      "query_string": {
+                        "query": "event.dataset: kafka.log"
                       }
                     }
                   ],
@@ -86,7 +140,7 @@
                 "range": {
                   "@timestamp": {
                     "format": "strict_date_optional_time",
-                    "gte": "{{bulk_start_date}}T21:52:08.615Z",
+                    "gte": "{{bulk_start_date}}T09:10:05.477Z",
                     "lte": "{{bulk_end_date}}T21:52:08.615Z"
                   }
                 }
@@ -96,13 +150,29 @@
             "must_not": []
           }
         },
-        "size": 500
+        "size": 500,
+        "sort": [
+          {
+            "@timestamp": {
+              "order": "desc"
+            }
+          }
+        ]
       }
     },
     {
       "name": "qstr_esql_query",
       "operation-type": "esql",
-      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE qstr(\"slack\", {\"default_field\": \"*\"}) | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | LIMIT 500"
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE QSTR(\"slack\") | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T21:52:08.615Z",
+            "lte": "{{bulk_end_date}}T21:52:08.615Z"
+          }
+        }
+      }
     },
     {
       "name": "qstr_query_dsl_query",
@@ -132,13 +202,43 @@
             "must_not": []
           }
         },
-        "size": 500
+        "size": 500,
+        "sort": [
+          {
+            "@timestamp": {
+              "order": "desc"
+            }
+          }
+        ]
       }
     },
     {
       "name": "kql_esql_query",
       "operation-type": "esql",
-      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE kql(\"query\") | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | LIMIT 500"
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE kql(\"query\") | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T07:52:08.615Z",
+            "lte": "{{bulk_end_date}}T07:52:08.615Z"
+          }
+        }
+      }
+    },
+    {
+      "name": "kql_like_esql_query",
+      "operation-type": "esql",
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE message like \"*query*\" | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T07:52:08.615Z",
+            "lte": "{{bulk_end_date}}T07:52:08.615Z"
+          }
+        }
+      }
     },
     {
       "name": "kql_query_dsl_query",
@@ -168,20 +268,49 @@
             "must_not": []
           }
         },
-        "size": 500
+        "size": 500,
+        "sort": [
+          {
+            "@timestamp": {
+              "order": "desc"
+            }
+          }
+        ]
       }
     },
     {
       "name": "syslog_qstr_esql_query",
       "operation-type": "esql",
-      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset: \"system.syslog\" AND qstr(\"Stopped*\") | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | LIMIT 100"
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE QSTR(\"event.dataset: system.syslog\") AND QSTR(\"Stopped*\") | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T20:13:05.675Z",
+            "lte": "{{bulk_end_date}}T20:13:05.675Z"
+          }
+        }
+      }
+    },
+    {
+      "name": "syslog_like_esql_query",
+      "operation-type": "esql",
+      "query": "FROM {{p_esql_target_prefix}}logs-* | WHERE event.dataset == \"system.syslog\" AND message LIKE \"Stopped*\" | SORT @timestamp DESC | LIMIT 500",
+      "filter": {
+        "range": {
+          "@timestamp": {
+            "format": "strict_date_optional_time",
+            "gte": "{{bulk_start_date}}T20:13:05.675Z",
+            "lte": "{{bulk_end_date}}T20:13:05.675Z"
+          }
+        }
+      }
     },
     {
       "name": "syslog_query_dsl_query",
       "operation-type": "search",
       "index": "{{p_esql_target_prefix}}logs-*",
       "body": {
-        "size": 100,
         "query": {
            "bool": {
             "filter": [
@@ -192,8 +321,8 @@
                       "bool": {
                         "should": [
                           {
-                            "match": {
-                              "event.dataset": "system.syslog"
+                            "query_string": {
+                              "query": "event.dataset: system.syslog"
                             }
                           }
                         ],
@@ -228,6 +357,14 @@
             "should": [],
             "must_not": []
           }
-        }
+        },
+        "size": 500,
+        "sort": [
+          {
+            "@timestamp": {
+              "order": "desc"
+            }
+          }
+        ]
       }
     }

wikipedia/track.py

@@ -238,7 +238,7 @@ def params(self):
                 raise ValueError("Unknown query type: " + self._query_type)
 
             return {
-                "query": f"FROM {self._index_name} METADATA _score | WHERE { query_body } | KEEP title, _score | SORT _score DESC | LIMIT { self._size }",
+                "query": f"FROM {self._index_name} METADATA _score | WHERE { query_body } | SORT _score DESC | LIMIT { self._size }",
             }
 
         except StopIteration: