From 0a1825bc6ab728d0121fbfda27ef28988f2b60ed Mon Sep 17 00:00:00 2001 From: favilo Date: Fri, 12 Apr 2024 11:13:31 -0700 Subject: [PATCH 01/14] Adding more ESQL queries for scale testing --- .../logging-querying-many-shards.json | 292 ++++++++++++++++++ elastic/logs/operations/esql.json | 15 + 2 files changed, 307 insertions(+) create mode 100644 elastic/logs/challenges/logging-querying-many-shards.json diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json new file mode 100644 index 000000000..b41d6a8c3 --- /dev/null +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -0,0 +1,292 @@ +{% import "rally.helpers" as rally %} +{ + "name": "logging-querying", + "description": "Applies a query workload. Ensures data streams exist so queries can be run, but does not remove existing data.", + "parameters": { + "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }} + }, + "schedule": [ + {% include "tasks/index-setup.json" %}, + {% if bulk_start_date and bulk_end_date %} + { + "name": "bulk-index", + "operation": { + "operation-type": "raw-bulk", + "param-source": "processed-source", + "time-format": "milliseconds", + "profile": "fixed_interval", + "init-load": true, + "bulk-size": {{ p_bulk_size }}, + "detailed-results": true + }, + "clients": {{ p_bulk_indexing_clients }}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}" + } + {%- if force_merge_max_num_segments is defined %}, + { + "name": "refresh-after-index", + "index": "logs-*", + "operation": "refresh" + }, + { + "name": "wait-until-index-merges-fininshes", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "operation": { + "operation-type": "force-merge", + "index": "logs-*", + "request-timeout": 36000, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + } + }, + { + "name": "wait-until-merges-finish", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "name": "refresh-after-force-merge", + "index": "logs-*", + "operation": "refresh" + } + {%- endif %} + {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}, + { + "name": "compression-stats", + "operation": { + "operation-type": "compression-statistics", + "param-source": "create-datastream-source" + } + } + {%- endif -%}{# non-serverless-index-statistics-marker-end #} + , + {% endif %} + { + "name": "logging-queries", + "parallel": { + "time-period": {{ p_query_time_period }}, + "warmup-time-period": {{ p_query_warmup_time_period }}, + "tasks": [ + {% for workflow in p_query_workflows %} + { + "name": {{workflow | tojson }}, + "operation": { + "operation-type": "composite", + "param-source": "workflow-selector", + "workflow": {{workflow | tojson }}, + "task-offset": {{ loop.index }}, + "request-params": {{ p_query_request_params | tojson(indent=2) }} + }, + "think-time-interval": {{ p_user_think_time }}, + "workflow-interval": {{ p_user_workflow_time }}, + "clients": {{ p_search_clients }}, + "schedule": "workflow-scheduler" + }{{ ", " if not loop.last else "" }} + {% endfor %} + ] + } + }{%- if p_include_esql_queries %}, + { + "operation": "esql_basic_count_group_0_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_1_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_2_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_3_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_4_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_dissect_duration_and_stats_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_1", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_2", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_3", + "clients": 1, + "warmup-iterations": 5, + "iterations": 10, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_4", + "clients": 1, + "warmup-iterations": 5, + "iterations": 10, + "tags": ["esql", "count"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "date_histogram"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_post_filter", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "date_histogram"] + }, + { + "operation": "esql_dissect_duration_and_stats", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "dissect"] + }, + { + "operation": "esql_from_all_limit_50", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "limit50"] + }, + { + "operation": "esql_from_logs_limit_50", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "limit50"] + }, + { + "operation": "esql_from_logs_limit_50", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "limit50"] + }, + { + "operation": "esql_topn_from_index_keep", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "limit50"] + }{%- if build_flavor != "serverless" or serverless_operator == true %}, + { + "operation": "disable_query_cache", + "tags": ["esql", "settings"] + }, + { + "operation": "search_basic_count_group_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_1", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_2", + "clients": 1, + "warmup-iterations": 2, + "iterations": 5, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_3", + "clients": 1, + "warmup-iterations": 1, + "iterations": 3, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_4", + "clients": 1, + "warmup-iterations": 1, + "iterations": 3, + "tags": ["esql", "count", "search"] + }, + { + "operation": "enable_query_cache", + "tags": ["esql", "settings"] + }{%- endif %} + {%- endif %} + ] +} diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 2eaa1b711..43c198a74 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -199,4 +199,19 @@ "name": "esql_from_all_limit_50", "operation-type": "esql", "query": "FROM * | LIMIT 50" + }, + { + "name": "esql_from_logs_limit_50", + "operation-type": "esql", + "query": "FROM logs-* | LIMIT 50" + }, + { + "name": "esql_from_logs-postgres_limit_50", + "operation-type": "esql", + "query": "FROM logs-postgres* | LIMIT 50" + }, + { + "name": "esql_topn_from_index_keep", + "operation-type": "esql", + "query": "topN (FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp, id)" } From 9ecc0934ffcf8b3534ec84950a43bc4799ed09fe Mon Sep 17 00:00:00 2001 From: favilo Date: Fri, 12 Apr 2024 12:13:48 -0700 Subject: [PATCH 02/14] Rename operations --- elastic/logs/challenges/logging-querying-many-shards.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index b41d6a8c3..f5f6a3bf0 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -231,7 +231,7 @@ "tags": ["esql", "limit50"] }, { - "operation": "esql_from_logs_limit_50", + "operation": "esql_from_logs_postgres_limit_50", "clients": 1, "warmup-iterations": 5, "iterations": 20, @@ -242,7 +242,7 @@ "clients": 1, "warmup-iterations": 5, "iterations": 20, - "tags": ["esql", "limit50"] + "tags": ["esql", "topn"] }{%- if build_flavor != "serverless" or serverless_operator == true %}, { "operation": "disable_query_cache", From 4728c138dba14ab18bd56f4659c689a13095dd38 Mon Sep 17 00:00:00 2001 From: favilo Date: Fri, 12 Apr 2024 12:18:34 -0700 Subject: [PATCH 03/14] Change challenge name --- elastic/logs/challenges/logging-querying-many-shards.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index f5f6a3bf0..549a4be3c 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -1,6 +1,6 @@ {% import "rally.helpers" as rally %} { - "name": "logging-querying", + "name": "logging-querying-many-shards", "description": "Applies a query workload. Ensures data streams exist so queries can be run, but does not remove existing data.", "parameters": { "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }} From 63376103d43f408501f895d014a3516a660bd5a9 Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 16 Apr 2024 17:50:21 -0700 Subject: [PATCH 04/14] Allow up to 50K shards per node --- elastic/logs/challenges/logging-querying-many-shards.json | 1 + 1 file changed, 1 insertion(+) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 549a4be3c..0cf6e8778 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -6,6 +6,7 @@ "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }} }, "schedule": [ + {% include "tasks/many-shards-setup.json" %}, {% include "tasks/index-setup.json" %}, {% if bulk_start_date and bulk_end_date %} { From d139eca90207d978690af14c6864080538b91c9a Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 16 Apr 2024 19:43:21 -0700 Subject: [PATCH 05/14] Delete duplicate setup task json entry --- elastic/logs/challenges/logging-querying-many-shards.json | 1 - 1 file changed, 1 deletion(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 0cf6e8778..5c806383d 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -7,7 +7,6 @@ }, "schedule": [ {% include "tasks/many-shards-setup.json" %}, - {% include "tasks/index-setup.json" %}, {% if bulk_start_date and bulk_end_date %} { "name": "bulk-index", From c9d49ee2318fe7fe455cf9f314984df4bf514152 Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 30 Apr 2024 09:41:35 -0700 Subject: [PATCH 06/14] Rename badly named operation --- elastic/logs/operations/esql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 43c198a74..644819801 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -206,7 +206,7 @@ "query": "FROM logs-* | LIMIT 50" }, { - "name": "esql_from_logs-postgres_limit_50", + "name": "esql_from_logs_postgres_limit_50", "operation-type": "esql", "query": "FROM logs-postgres* | LIMIT 50" }, From a45a9dd8dc0ce7203954ebee309269c2cf55b0bf Mon Sep 17 00:00:00 2001 From: favilo Date: Mon, 6 May 2024 13:29:35 -0700 Subject: [PATCH 07/14] Add tag to wait opperation to allow us to add wait for green to filter --- elastic/logs/challenges/logging-querying-many-shards.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 5c806383d..36853cf22 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -39,7 +39,8 @@ }, "retry-until-success": true, "include-in-reporting": false - } + }, + "tags": ["wait"] }, { "operation": { From 153a611d5e9ea262a3e115e74ed60a28614b54dc Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 7 May 2024 14:33:25 -0700 Subject: [PATCH 08/14] Drop topN, since it was causing errors --- elastic/logs/operations/esql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 644819801..3c7391cb1 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -213,5 +213,5 @@ { "name": "esql_topn_from_index_keep", "operation-type": "esql", - "query": "topN (FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp, id)" + "query": "FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp, id" } From 29b4a822e79e3d2dafa5404f4172ce94f08889df Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 14 May 2024 13:46:48 -0700 Subject: [PATCH 09/14] Remove un-needed queries --- .../logging-querying-many-shards.json | 112 ------------------ 1 file changed, 112 deletions(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 36853cf22..5a2c0b607 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -105,118 +105,6 @@ ] } }{%- if p_include_esql_queries %}, - { - "operation": "esql_basic_count_group_0_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_1_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_2_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_3_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_4_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_dissect_duration_and_stats_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_1", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_2", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_3", - "clients": 1, - "warmup-iterations": 5, - "iterations": 10, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_4", - "clients": 1, - "warmup-iterations": 5, - "iterations": 10, - "tags": ["esql", "count"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "date_histogram"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_post_filter", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "date_histogram"] - }, - { - "operation": "esql_dissect_duration_and_stats", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "dissect"] - }, { "operation": "esql_from_all_limit_50", "clients": 1, From 7b3187845299bfbf5398d97e35015d58ca41657d Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 14 May 2024 16:16:57 -0700 Subject: [PATCH 10/14] Remove `id` from topN query. We need to find a new to use --- elastic/logs/operations/esql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 3c7391cb1..993779d25 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -213,5 +213,5 @@ { "name": "esql_topn_from_index_keep", "operation-type": "esql", - "query": "FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp, id" + "query": "FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp" } From 7363ba0d7c76afcf2ae2cca267338048331c7314 Mon Sep 17 00:00:00 2001 From: favilo Date: Tue, 14 May 2024 16:21:50 -0700 Subject: [PATCH 11/14] Increase the number of iterations we run --- .../challenges/logging-querying-many-shards.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 5a2c0b607..1cbb4628d 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -108,29 +108,29 @@ { "operation": "esql_from_all_limit_50", "clients": 1, - "warmup-iterations": 5, - "iterations": 20, + "warmup-iterations": 10, + "iterations": 50, "tags": ["esql", "limit50"] }, { "operation": "esql_from_logs_limit_50", "clients": 1, - "warmup-iterations": 5, - "iterations": 20, + "warmup-iterations": 10, + "iterations": 50, "tags": ["esql", "limit50"] }, { "operation": "esql_from_logs_postgres_limit_50", "clients": 1, - "warmup-iterations": 5, - "iterations": 20, + "warmup-iterations": 10, + "iterations": 50, "tags": ["esql", "limit50"] }, { "operation": "esql_topn_from_index_keep", "clients": 1, - "warmup-iterations": 5, - "iterations": 20, + "warmup-iterations": 10, + "iterations": 50, "tags": ["esql", "topn"] }{%- if build_flavor != "serverless" or serverless_operator == true %}, { From 409494ea014e972123df8b57a77a819c317a48e5 Mon Sep 17 00:00:00 2001 From: favilo Date: Thu, 13 Jun 2024 16:31:18 -0700 Subject: [PATCH 12/14] Adding ES|QL queries from spreadsheet [spreadsheet] [spreadsheet]: https://docs.google.com/spreadsheets/d/1sZ_0kiGVneAzLEEq-p_o9bafQzj5uSgoFgfvTpaTSq8/edit#gid=0 --- .../logging-querying-many-shards.json | 59 +++++++++---------- elastic/logs/operations/esql.json | 47 ++++++++++++++- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 1cbb4628d..9d30d2a47 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -106,76 +106,75 @@ } }{%- if p_include_esql_queries %}, { - "operation": "esql_from_all_limit_50", + "operation": "esql_from_logs_limit_50", "clients": 1, "warmup-iterations": 10, "iterations": 50, "tags": ["esql", "limit50"] }, { - "operation": "esql_from_logs_limit_50", + "operation": "esql_avg_doc_size", "clients": 1, "warmup-iterations": 10, "iterations": 50, - "tags": ["esql", "limit50"] + "tags": ["esql", "avg"] }, { - "operation": "esql_from_logs_postgres_limit_50", + "operation": "esql_avg_compression", "clients": 1, "warmup-iterations": 10, "iterations": 50, - "tags": ["esql", "limit50"] + "tags": ["esql", "avg"] }, { - "operation": "esql_topn_from_index_keep", + "operation": "esql_avg_amount_group_by_integer", "clients": 1, "warmup-iterations": 10, "iterations": 50, - "tags": ["esql", "topn"] - }{%- if build_flavor != "serverless" or serverless_operator == true %}, - { - "operation": "disable_query_cache", - "tags": ["esql", "settings"] + "tags": ["esql", "avg", "groupby"] }, { - "operation": "search_basic_count_group_0", + "operation": "esql_avg_amount_group_by_keyword", "clients": 1, "warmup-iterations": 10, "iterations": 50, - "tags": ["esql", "count", "search"] + "tags": ["esql", "avg", "groupby"] }, { - "operation": "search_basic_count_group_1", + "operation": "esql_count_group_by_keyword", "clients": 1, "warmup-iterations": 10, "iterations": 50, - "tags": ["esql", "count", "search"] + "tags": ["esql", "count", "groupby"] }, { - "operation": "search_basic_count_group_2", + "operation": "esql_sort_by_ts", "clients": 1, - "warmup-iterations": 2, - "iterations": 5, - "tags": ["esql", "count", "search"] + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "sort"] }, { - "operation": "search_basic_count_group_3", + "operation": "esql_date_histogram_fixed_interval", "clients": 1, - "warmup-iterations": 1, - "iterations": 3, - "tags": ["esql", "count", "search"] + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] }, { - "operation": "search_basic_count_group_4", + "operation": "esql_date_histogram_fixed_interval_with_metrics", "clients": 1, - "warmup-iterations": 1, - "iterations": 3, - "tags": ["esql", "count", "search"] + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] }, { - "operation": "enable_query_cache", - "tags": ["esql", "settings"] - }{%- endif %} + "operation": "esql_multi_terms_keyword", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] + } {%- endif %} ] } diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 993779d25..f96e09908 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -214,4 +214,49 @@ "name": "esql_topn_from_index_keep", "operation-type": "esql", "query": "FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp" - } + }, + { + "name": "esql_avg_doc_size", + "operation-type": "esql", + "query": "FROM logs-* | STATS AVG(rally.doc_size)" + }, + { + "name": "esql_avg_compression", + "operation-type": "esql", + "query": "FROM logs-* | EVAL compression = rally.doc_size::DOUBLE / rally.message_size | STATS AVG(compression)" + }, + { + "name": "esql_avg_amount_group_by_integer", + "operation-type": "esql", + "query": "FROM logs-* | STATS AVG(rally.message_size) BY b = BUCKET(rally.doc_size, 1000.) | SORT b" + }, + { + "name": "esql_avg_amount_group_by_keyword", + "operation-type": "esql", + "query": "FROM logs-* | STATS AVG(rally.message_size) BY cloud.machine.type" + }, + { + "name": "esql_count_group_by_keyword", + "operation-type": "esql", + "query": "FROM logs-* | STATS count(*) BY cloud.machine.type" + }, + { + "name": "esql_sort_by_ts", + "operation-type": "esql", + "query": "FROM logs-* | SORT @timestamp DESC | KEEP @timestamp, host.name, log.offset | LIMIT 1000" + }, + { + "name": "esql_date_histogram_fixed_interval", + "operation-type": "esql", + "query": "FROM logs-* | STATS COUNT(*) BY b=BUCKET(@timestamp, 1 WEEK) | SORT b" + }, + { + "name": "esql_date_histogram_fixed_interval_with_metrics", + "operation-type": "esql", + "query": "FROM logs-* | EVAL every10days = DATE_TRUNC(10 DAYS, @timestamp) | STATS min=MIN(log.offset), max=MAX(log.offset), avg=AVG(log.offset) BY every10days | SORT every10days" + }, + { + "name": "esql_multi_terms_keyword", + "operation-type": "esql", + "query": "FROM logs-* | STATS c=COUNT(@timestamp) BY cloud.machine.type, cloud.provider, cloud.region" + }, From cb95a5b675eff17c2aacfac293e58c0fbddd2c8d Mon Sep 17 00:00:00 2001 From: favilo Date: Fri, 21 Jun 2024 16:44:13 -0700 Subject: [PATCH 13/14] Remove unused queries. Remove trailing comma --- elastic/logs/operations/esql.json | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index f96e09908..0c7ea2d18 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -195,26 +195,11 @@ "operation-type": "esql", "query": "FROM logs-postgres* | DISSECT message \"duration: %{query_duration} ms\" | EVAL query_duration_num = TO_DOUBLE(query_duration) | STATS avg_duration = AVG(query_duration_num) | LIMIT 0" }, - { - "name": "esql_from_all_limit_50", - "operation-type": "esql", - "query": "FROM * | LIMIT 50" - }, { "name": "esql_from_logs_limit_50", "operation-type": "esql", "query": "FROM logs-* | LIMIT 50" }, - { - "name": "esql_from_logs_postgres_limit_50", - "operation-type": "esql", - "query": "FROM logs-postgres* | LIMIT 50" - }, - { - "name": "esql_topn_from_index_keep", - "operation-type": "esql", - "query": "FROM logs-* | SORT @timestamp | LIMIT 1000 | KEEP @timestamp" - }, { "name": "esql_avg_doc_size", "operation-type": "esql", @@ -259,4 +244,4 @@ "name": "esql_multi_terms_keyword", "operation-type": "esql", "query": "FROM logs-* | STATS c=COUNT(@timestamp) BY cloud.machine.type, cloud.provider, cloud.region" - }, + } From 3747188c588884c41c998f8de705c747e8f5f958 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 25 Jun 2024 18:37:43 +0200 Subject: [PATCH 14/14] Add time filter and reduce iterations --- .../challenges/logging-querying-many-shards.json | 16 ++++++++-------- elastic/logs/operations/esql.json | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/elastic/logs/challenges/logging-querying-many-shards.json b/elastic/logs/challenges/logging-querying-many-shards.json index 9d30d2a47..506462f0e 100644 --- a/elastic/logs/challenges/logging-querying-many-shards.json +++ b/elastic/logs/challenges/logging-querying-many-shards.json @@ -115,29 +115,29 @@ { "operation": "esql_avg_doc_size", "clients": 1, - "warmup-iterations": 10, - "iterations": 50, + "warmup-iterations": 1, + "iterations": 1, "tags": ["esql", "avg"] }, { "operation": "esql_avg_compression", "clients": 1, - "warmup-iterations": 10, - "iterations": 50, + "warmup-iterations": 1, + "iterations": 1, "tags": ["esql", "avg"] }, { "operation": "esql_avg_amount_group_by_integer", "clients": 1, - "warmup-iterations": 10, - "iterations": 50, + "warmup-iterations": 1, + "iterations": 1, "tags": ["esql", "avg", "groupby"] }, { "operation": "esql_avg_amount_group_by_keyword", "clients": 1, - "warmup-iterations": 10, - "iterations": 50, + "warmup-iterations": 1, + "iterations": 1, "tags": ["esql", "avg", "groupby"] }, { diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 0c7ea2d18..9b22dd129 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -203,22 +203,22 @@ { "name": "esql_avg_doc_size", "operation-type": "esql", - "query": "FROM logs-* | STATS AVG(rally.doc_size)" + "query": "FROM logs-* | WHERE @timestamp >= \"2020-08-17T00:00:00.000Z\"::DATETIME | STATS AVG(rally.doc_size)" }, { "name": "esql_avg_compression", "operation-type": "esql", - "query": "FROM logs-* | EVAL compression = rally.doc_size::DOUBLE / rally.message_size | STATS AVG(compression)" + "query": "FROM logs-* | WHERE @timestamp >= \"2020-08-17T00:00:00.000Z\"::DATETIME | EVAL compression = rally.doc_size::DOUBLE / rally.message_size | STATS AVG(compression)" }, { "name": "esql_avg_amount_group_by_integer", "operation-type": "esql", - "query": "FROM logs-* | STATS AVG(rally.message_size) BY b = BUCKET(rally.doc_size, 1000.) | SORT b" + "query": "FROM logs-* | WHERE @timestamp >= \"2020-08-17T00:00:00.000Z\"::DATETIME | STATS AVG(rally.message_size) BY b = BUCKET(rally.doc_size, 1000.) | SORT b" }, { "name": "esql_avg_amount_group_by_keyword", "operation-type": "esql", - "query": "FROM logs-* | STATS AVG(rally.message_size) BY cloud.machine.type" + "query": "FROM logs-* | WHERE @timestamp >= \"2020-08-17T00:00:00.000Z\"::DATETIME | STATS AVG(rally.message_size) BY cloud.machine.type" }, { "name": "esql_count_group_by_keyword",