From b8d09fa27eac035dc36a75f057ebd01f70b18467 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 17:59:45 +0100 Subject: [PATCH 01/19] Make base path configuration conditional --- elastic/logs/challenges/esql-ccs-snapshot.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index c8f4f52bd..932690610 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -49,7 +49,9 @@ {# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #} {% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %} {# randomize base path to avoid clashes when running concurrent benchmarks #} - {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} + {% if 'base_path' not in many_clusters_snapshot_repo_setting %} + {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} + {% endif %} { "name": "register-snapshot-repository-on-local-cluster", "operation": { From 36a237bf782706ae2a669e3408b72fbe372826ef Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 18:23:37 +0100 Subject: [PATCH 02/19] Fix typo --- elastic/logs/challenges/esql-ccs-snapshot.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 932690610..9b8858010 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -49,7 +49,7 @@ {# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #} {% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %} {# randomize base path to avoid clashes when running concurrent benchmarks #} - {% if 'base_path' not in many_clusters_snapshot_repo_setting %} + {% if 'base_path' not in many_clusters_snapshot_repo_settings %} {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} {% endif %} { From 304849dc39a8301dc5b330ee4ca2c7d14eb7f44f Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 19:19:18 +0100 Subject: [PATCH 03/19] Refactor to control local snapshot --- .../logs/challenges/esql-ccs-snapshot.json | 82 ++++++++++--------- elastic/logs/track.json | 1 + 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 9b8858010..448916cbe 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -32,26 +32,28 @@ "request-timeout": 60 } }, - {% if p_skip_esql_ccs_remote_clusters %} - { - "name": "check-cluster-health-local", - "operation": { - "operation-type": "cluster-health", - "index": "_all", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - {% else %}{# if p_skip_esql_ccs_remote_clusters ... #} - {# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #} - {% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %} - {# randomize base path to avoid clashes when running concurrent benchmarks #} - {% if 'base_path' not in many_clusters_snapshot_repo_settings %} - {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} - {% endif %} + + {# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #} + {% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %} + {# randomize base path to avoid clashes when running concurrent benchmarks #} + {% if 'base_path' not in many_clusters_snapshot_repo_settings %} + {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} + {% endif %} + + { + "name": "check-cluster-health-on-local-cluster", + "operation": { + "operation-type": "cluster-health", + "index": "_all", + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } + }, + + {% if p_esql_ccs_include_local_snapshot %} { "name": "register-snapshot-repository-on-local-cluster", "operation": { @@ -63,20 +65,6 @@ } } }, - {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} - { - "name": "register-snapshot-repository-on-remote-clusters", - "operation": { - "operation-type": "multi-cluster-wrapper", - "base-operation-type": "create-snapshot-repository", - "ignore-clusters": ["default"], - "repository": {{ p_snapshot_repo_name | tojson }}, - "body": { - "type": {{ p_snapshot_repo_type | tojson }}, - "settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}} - } - } - }, { "name": "create-snapshot-{{ p_snapshot_name }}", "operation": { @@ -101,6 +89,23 @@ "repository": {{ p_snapshot_repo_name | tojson }} } }, + {% endif %} + + {% if not p_skip_esql_ccs_remote_clusters %} + {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} + { + "name": "register-snapshot-repository-on-remote-clusters", + "operation": { + "operation-type": "multi-cluster-wrapper", + "base-operation-type": "create-snapshot-repository", + "ignore-clusters": ["default"], + "repository": {{ p_snapshot_repo_name | tojson }}, + "body": { + "type": {{ p_snapshot_repo_type | tojson }}, + "settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}} + } + } + }, { "name": "recover-snapshot-{{ p_snapshot_name }}-on-remote-clusters", "operation": { @@ -120,18 +125,20 @@ } }, { - "name": "wait-for-snapshot-recovery-{{ p_snapshot_name }}-on-all-clusters", + "name": "wait-for-snapshot-recovery-{{ p_snapshot_name }}-on-remote-clusters", "operation": { "operation-type": "multi-cluster-wrapper", "base-operation-type": "wait-for-recovery", + "ignore-clusters": ["default"], "index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }} } }, { - "name": "check-cluster-health-all-clusters", + "name": "check-cluster-health-on-remote-clusters", "operation": { "operation-type": "multi-cluster-wrapper", "base-operation-type": "cluster-health", + "ignore-clusters": ["default"], "index": "_all", "request-params": { "wait_for_status": "{{wait_for_status | default('green')}}", @@ -163,7 +170,8 @@ } } }, - {% endif %}{# if p_skip_esql_ccs_remote_clusters ... #} + {% endif %}{# if not p_skip_esql_ccs_remote_clusters ... #} + {% include "tasks/many-clusters-esql-ccs-queries.json" %} ] } diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 228845513..b430e22cd 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -35,6 +35,7 @@ {% else %} {% set p_esql_target_prefix = "remote*:" %} {% endif %} +{% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_skip_esql_ccs_remote_clusters)) %} {% set p_snapshot_repo_name = (snapshot_repo_name | default("logging")) %} {% set p_snapshot_repo_type = (snapshot_repo_type | default("s3")) %} {% set p_snapshot_repo_settings = (snapshot_repo_settings | default({ From 88550c38ca15c867504073486725fa4211d5991b Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 19:24:49 +0100 Subject: [PATCH 04/19] Rename p_skip_esql_ccs_remote_clusters to p_esql_ccs_skip_remote_clusters --- elastic/logs/challenges/esql-ccs-snapshot.json | 6 +++--- elastic/logs/operations/esql.json | 6 +++--- elastic/logs/track.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 448916cbe..794d00bbe 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -3,7 +3,7 @@ "name": "esql-ccs-snapshot", "description": "Indexes logs to the local cluster, and runs ESQL queries either locally or across remote clusters via CCS. Data replicated via snapshot/restore.", "schedule": [ - {% if not p_skip_esql_ccs_remote_clusters %} + {% if not p_esql_ccs_skip_remote_clusters %} {% include "tasks/many-clusters-setup.json" %}, {% endif %} {% include "tasks/index-setup.json" %}, @@ -91,7 +91,7 @@ }, {% endif %} - {% if not p_skip_esql_ccs_remote_clusters %} + {% if not p_esql_ccs_skip_remote_clusters %} {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} { "name": "register-snapshot-repository-on-remote-clusters", @@ -170,7 +170,7 @@ } } }, - {% endif %}{# if not p_skip_esql_ccs_remote_clusters ... #} + {% endif %}{# if not p_esql_ccs_skip_remote_clusters ... #} {% include "tasks/many-clusters-esql-ccs-queries.json" %} ] diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index e2a13d9e1..447fca770 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -1,7 +1,7 @@ { "name": "enable_query_cache", "include-in-reporting": false, - {% if p_skip_esql_ccs_remote_clusters %} + {% if p_esql_ccs_skip_remote_clusters %} "operation-type": "raw-request", {% else %} "operation-type": "multi-cluster-wrapper", @@ -17,7 +17,7 @@ { "name": "disable_query_cache", "include-in-reporting": false, - {% if p_skip_esql_ccs_remote_clusters %} + {% if p_esql_ccs_skip_remote_clusters %} "operation-type": "raw-request", {% else %} "operation-type": "multi-cluster-wrapper", @@ -33,7 +33,7 @@ { "name": "restore_query_cache_default", "include-in-reporting": false, - {% if p_skip_esql_ccs_remote_clusters %} + {% if p_esql_ccs_skip_remote_clusters %} "operation-type": "raw-request", {% else %} "operation-type": "multi-cluster-wrapper", diff --git a/elastic/logs/track.json b/elastic/logs/track.json index b430e22cd..eec15e7c2 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -29,13 +29,13 @@ {% set p_recovery_max_bytes_per_sec = (recovery_max_bytes_per_sec | default(-1)) %} {% set p_node_concurrent_recoveries = (node_concurrent_recoveries | default(2)) %} -{% set p_skip_esql_ccs_remote_clusters = (skip_esql_ccs_remote_clusters | default(true)) %} -{% if p_skip_esql_ccs_remote_clusters %} +{% set p_esql_ccs_skip_remote_clusters = (esql_ccs_skip_remote_clusters | default(true)) %} +{% if p_esql_ccs_skip_remote_clusters %} {% set p_esql_target_prefix = "" %} {% else %} {% set p_esql_target_prefix = "remote*:" %} {% endif %} -{% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_skip_esql_ccs_remote_clusters)) %} +{% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_esql_ccs_skip_remote_clusters)) %} {% set p_snapshot_repo_name = (snapshot_repo_name | default("logging")) %} {% set p_snapshot_repo_type = (snapshot_repo_type | default("s3")) %} {% set p_snapshot_repo_settings = (snapshot_repo_settings | default({ From 13fd2f220755e18b7eeec4b1db7b6b1cf0609c42 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 19:57:53 +0100 Subject: [PATCH 05/19] Add newlines for clarity --- elastic/logs/challenges/esql-ccs-snapshot.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 794d00bbe..a9ab1b5a2 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -3,9 +3,11 @@ "name": "esql-ccs-snapshot", "description": "Indexes logs to the local cluster, and runs ESQL queries either locally or across remote clusters via CCS. Data replicated via snapshot/restore.", "schedule": [ + {% if not p_esql_ccs_skip_remote_clusters %} {% include "tasks/many-clusters-setup.json" %}, {% endif %} + {% include "tasks/index-setup.json" %}, { "name": "bulk-index", From 8b95300770d09d70a42176214b4fd6cbf028a7e1 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 5 Nov 2024 20:33:31 +0100 Subject: [PATCH 06/19] Add track parameter to control data generation --- elastic/logs/challenges/esql-ccs-snapshot.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index a9ab1b5a2..acabb99fa 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -2,6 +2,9 @@ { "name": "esql-ccs-snapshot", "description": "Indexes logs to the local cluster, and runs ESQL queries either locally or across remote clusters via CCS. Data replicated via snapshot/restore.", + "parameters": { + "generate-data": {{ generate_data | default('true') | tojson }} + }, "schedule": [ {% if not p_esql_ccs_skip_remote_clusters %} From 7538bff39c352235035742bdd4f6f09c6e90d075 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Wed, 6 Nov 2024 08:52:52 +0100 Subject: [PATCH 07/19] Remove snapshot name from task names --- elastic/logs/challenges/esql-ccs-snapshot.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index acabb99fa..a41bfcd97 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -71,7 +71,7 @@ } }, { - "name": "create-snapshot-{{ p_snapshot_name }}", + "name": "create-snapshot", "operation": { "operation-type": "create-snapshot", "repository": {{ p_snapshot_repo_name | tojson }}, @@ -87,7 +87,7 @@ } }, { - "name": "wait-for-snapshot-{{ p_snapshot_name }}", + "name": "wait-for-snapshot", "operation": { "operation-type": "wait-for-snapshot-create", "snapshot": {{ p_snapshot_name | tojson }}, @@ -112,7 +112,7 @@ } }, { - "name": "recover-snapshot-{{ p_snapshot_name }}-on-remote-clusters", + "name": "recover-snapshot-on-remote-clusters", "operation": { "operation-type": "multi-cluster-wrapper", "base-operation-type": "restore-snapshot", @@ -130,7 +130,7 @@ } }, { - "name": "wait-for-snapshot-recovery-{{ p_snapshot_name }}-on-remote-clusters", + "name": "wait-for-snapshot-recovery-on-remote-clusters", "operation": { "operation-type": "multi-cluster-wrapper", "base-operation-type": "wait-for-recovery", From 16ca9b290ac6e0a13a5f3a1808f606e51addd65e Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Wed, 6 Nov 2024 13:08:47 +0100 Subject: [PATCH 08/19] Add scaling queries --- elastic/logs/operations/esql.json | 106 +++++ .../tasks/many-clusters-esql-ccs-queries.json | 407 +++++++++++------- elastic/logs/track.json | 4 + 3 files changed, 362 insertions(+), 155 deletions(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 447fca770..bfb912cb1 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -102,6 +102,25 @@ } } }, + { + "name": "search_basic_count_group_1_no_minimized_roundtrips", + "operation-type": "search", + "index": "{{p_esql_target_prefix}}logs-*", + "body": { + "size": 0, + "aggs": { + "count": { + "terms": { + "field": "agent.version", + "size": 20 + } + } + } + }, + "request-params": { + "ccs_minimize_roundtrips": false + } + }, { "name": "search_basic_count_group_2", "operation-type": "search", @@ -227,4 +246,91 @@ "name": "esql_from_all_limit_50", "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}* | LIMIT 50" + }, + {# ESQL scale testing #} + { + "name": "esql_from_logs_limit_50", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | LIMIT 50" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_avg_doc_size", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.doc_size)" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_avg_compression", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | EVAL compression = rally.doc_size::DOUBLE / rally.message_size | STATS AVG(compression)" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_avg_amount_group_by_integer", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.message_size) BY b = BUCKET(rally.doc_size, 1000.) | SORT b" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_avg_amount_group_by_keyword", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.message_size) BY cloud.machine.type" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_count_group_by_keyword", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS count(*) BY cloud.machine.type" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_sort_by_ts", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | SORT @timestamp DESC | KEEP @timestamp, host.name, log.offset | LIMIT 1000" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_date_histogram_fixed_interval", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS COUNT(*) BY b=BUCKET(@timestamp, 1 WEEK) | SORT b" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_date_histogram_fixed_interval_with_metrics", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | EVAL every10days = DATE_TRUNC(10 DAYS, @timestamp) | STATS min=MIN(log.offset), max=MAX(log.offset), avg=AVG(log.offset) BY every10days | SORT every10days" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "esql_multi_terms_keyword", + "operation-type": "esql", + "query": "FROM {{p_esql_target_prefix}}logs-* | STATS c=COUNT(@timestamp) BY cloud.machine.type, cloud.provider, cloud.region" + {%- if p_esql_ccs_extra_body %}, + "body": {{ p_esql_ccs_extra_body }} + {%- endif %} + }, + { + "name": "cluster-stats-with-remotes", + "operation-type": "raw-request", + "path": "/_cluster/stats?include_remotes=true", + "method": "GET" } diff --git a/elastic/logs/tasks/many-clusters-esql-ccs-queries.json b/elastic/logs/tasks/many-clusters-esql-ccs-queries.json index 8e81bbf80..8d009d83c 100644 --- a/elastic/logs/tasks/many-clusters-esql-ccs-queries.json +++ b/elastic/logs/tasks/many-clusters-esql-ccs-queries.json @@ -1,155 +1,252 @@ - { - "operation": "esql_basic_count_group_0_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_1_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_2_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_3_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_4_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_dissect_duration_and_stats_limit_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "limit0"] - }, - { - "operation": "esql_basic_count_group_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_1", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_2", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_3", - "clients": 1, - "warmup-iterations": 5, - "iterations": 10, - "tags": ["esql", "count"] - }, - { - "operation": "esql_basic_count_group_4", - "clients": 1, - "warmup-iterations": 5, - "iterations": 10, - "tags": ["esql", "count"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "date_histogram"] - }, - { - "operation": "esql_time_range_and_date_histogram_two_groups_post_filter", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "date_histogram"] - }, - { - "operation": "esql_dissect_duration_and_stats", - "clients": 1, - "warmup-iterations": 5, - "iterations": 20, - "tags": ["esql", "dissect"] - }{%- if build_flavor != "serverless" or serverless_operator == true %}, - { - "operation": "disable_query_cache", - "tags": ["esql", "settings"] - }, - { - "operation": "search_basic_count_group_0", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count", "search"] - }, - { - "operation": "search_basic_count_group_1", - "clients": 1, - "warmup-iterations": 10, - "iterations": 50, - "tags": ["esql", "count", "search"] - }, - { - "operation": "search_basic_count_group_2", - "clients": 1, - "warmup-iterations": 2, - "iterations": 5, - "tags": ["esql", "count", "search"] - }, - { - "operation": "search_basic_count_group_3", - "clients": 1, - "warmup-iterations": 1, - "iterations": 3, - "tags": ["esql", "count", "search"] - }, - { - "operation": "search_basic_count_group_4", - "clients": 1, - "warmup-iterations": 1, - "iterations": 3, - "tags": ["esql", "count", "search"] - }, - { - "operation": "restore_query_cache_default", - "tags": ["esql", "settings"] - }{%- endif %} + {% if not p_esql_ccs_scaling_queries %} + { + "operation": "esql_basic_count_group_0_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_1_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_2_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_3_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_4_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_dissect_duration_and_stats_limit_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit0"] + }, + { + "operation": "esql_basic_count_group_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_1", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_2", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_3", + "clients": 1, + "warmup-iterations": 5, + "iterations": 10, + "tags": ["esql", "count"] + }, + { + "operation": "esql_basic_count_group_4", + "clients": 1, + "warmup-iterations": 5, + "iterations": 10, + "tags": ["esql", "count"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_pre_filter", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "date_histogram"] + }, + { + "operation": "esql_time_range_and_date_histogram_two_groups_post_filter", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "date_histogram"] + }, + { + "operation": "esql_dissect_duration_and_stats", + "clients": 1, + "warmup-iterations": 5, + "iterations": 20, + "tags": ["esql", "dissect"] + } + {%- if build_flavor != "serverless" or serverless_operator == true %}, + { + "operation": "disable_query_cache", + "tags": ["esql", "settings"] + }, + { + "operation": "search_basic_count_group_0", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_1", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_2", + "clients": 1, + "warmup-iterations": 2, + "iterations": 5, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_3", + "clients": 1, + "warmup-iterations": 1, + "iterations": 3, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_4", + "clients": 1, + "warmup-iterations": 1, + "iterations": 3, + "tags": ["esql", "count", "search"] + }, + { + "operation": "restore_query_cache_default", + "tags": ["esql", "settings"] + } + {%- endif %}{# if build_flavor != "serverless" or serverless_operator == true #} + {% else %}{# if not p_esql_ccs_scaling_queries #} + { + "operation": "esql_from_logs_limit_50", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "limit50"] + }, + { + "operation": "esql_avg_doc_size", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "avg"] + }, + { + "operation": "esql_avg_compression", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "avg"] + }, + { + "operation": "esql_avg_amount_group_by_integer", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "avg", "groupby"] + }, + { + "operation": "esql_avg_amount_group_by_keyword", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "avg", "groupby"] + }, + { + "operation": "esql_count_group_by_keyword", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "groupby"] + }, + { + "operation": "esql_sort_by_ts", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "sort"] + }, + { + "operation": "esql_date_histogram_fixed_interval", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] + }, + { + "operation": "esql_date_histogram_fixed_interval_with_metrics", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] + }, + { + "operation": "esql_multi_terms_keyword", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "histogram"] + }, + {# CCS telemetry #} + { + "operation": "search_basic_count_group_1", + "clients": 1, + "warmup-iterations": 10, + "iterations": 50, + "tags": ["esql", "count", "search"] + }, + { + "operation": "search_basic_count_group_1_no_minimized_roundtrips", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "count", "search"] + }, + { + "operation": "cluster_stats_with_remotes", + "clients": 1, + "warmup-iterations": 1, + "iterations": 1, + "tags": ["esql", "stats"] + } + {% endif %}{# if notp_esql_ccs_scaling_queries #} diff --git a/elastic/logs/track.json b/elastic/logs/track.json index eec15e7c2..911ddc940 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -29,6 +29,7 @@ {% set p_recovery_max_bytes_per_sec = (recovery_max_bytes_per_sec | default(-1)) %} {% set p_node_concurrent_recoveries = (node_concurrent_recoveries | default(2)) %} + {% set p_esql_ccs_skip_remote_clusters = (esql_ccs_skip_remote_clusters | default(true)) %} {% if p_esql_ccs_skip_remote_clusters %} {% set p_esql_target_prefix = "" %} @@ -36,6 +37,9 @@ {% set p_esql_target_prefix = "remote*:" %} {% endif %} {% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_esql_ccs_skip_remote_clusters)) %} +{% set p_esql_ccs_scaling_queries = (esql_ccs_scaling_queries | default(false))) %} +{% set p_esql_ccs_extra_body = (esql_ccs_extra_body | default(""))) %} + {% set p_snapshot_repo_name = (snapshot_repo_name | default("logging")) %} {% set p_snapshot_repo_type = (snapshot_repo_type | default("s3")) %} {% set p_snapshot_repo_settings = (snapshot_repo_settings | default({ From d3656bac33e735ea457925e0c5a6cb9cbb67a0d6 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Wed, 6 Nov 2024 13:15:19 +0100 Subject: [PATCH 09/19] Remove extra parentheses --- elastic/logs/track.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 911ddc940..5b949f73d 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -37,8 +37,8 @@ {% set p_esql_target_prefix = "remote*:" %} {% endif %} {% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_esql_ccs_skip_remote_clusters)) %} -{% set p_esql_ccs_scaling_queries = (esql_ccs_scaling_queries | default(false))) %} -{% set p_esql_ccs_extra_body = (esql_ccs_extra_body | default(""))) %} +{% set p_esql_ccs_scaling_queries = (esql_ccs_scaling_queries | default(false)) %} +{% set p_esql_ccs_extra_body = (esql_ccs_extra_body | default("")) %} {% set p_snapshot_repo_name = (snapshot_repo_name | default("logging")) %} {% set p_snapshot_repo_type = (snapshot_repo_type | default("s3")) %} From 801c8b658eeac4b42c72bf6611188c0d76559d32 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Wed, 6 Nov 2024 13:30:29 +0100 Subject: [PATCH 10/19] Rename cluster stats operation --- elastic/logs/operations/esql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index bfb912cb1..becbda866 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -329,7 +329,7 @@ {%- endif %} }, { - "name": "cluster-stats-with-remotes", + "name": "cluster_stats_with_remotes", "operation-type": "raw-request", "path": "/_cluster/stats?include_remotes=true", "method": "GET" From ff9f236b1fde9f14c491f93cc560d014e56ef19a Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Wed, 6 Nov 2024 13:57:38 +0100 Subject: [PATCH 11/19] Add tojson filter --- elastic/logs/operations/esql.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index becbda866..0c8047f93 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -253,7 +253,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | LIMIT 50" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -261,7 +261,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.doc_size)" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -269,7 +269,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | EVAL compression = rally.doc_size::DOUBLE / rally.message_size | STATS AVG(compression)" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -277,7 +277,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.message_size) BY b = BUCKET(rally.doc_size, 1000.) | SORT b" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -285,7 +285,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS AVG(rally.message_size) BY cloud.machine.type" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -293,7 +293,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS count(*) BY cloud.machine.type" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -301,7 +301,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | SORT @timestamp DESC | KEEP @timestamp, host.name, log.offset | LIMIT 1000" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -309,7 +309,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS COUNT(*) BY b=BUCKET(@timestamp, 1 WEEK) | SORT b" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -317,7 +317,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | EVAL every10days = DATE_TRUNC(10 DAYS, @timestamp) | STATS min=MIN(log.offset), max=MAX(log.offset), avg=AVG(log.offset) BY every10days | SORT every10days" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { @@ -325,7 +325,7 @@ "operation-type": "esql", "query": "FROM {{p_esql_target_prefix}}logs-* | STATS c=COUNT(@timestamp) BY cloud.machine.type, cloud.provider, cloud.region" {%- if p_esql_ccs_extra_body %}, - "body": {{ p_esql_ccs_extra_body }} + "body": {{ p_esql_ccs_extra_body | tojson }} {%- endif %} }, { From e0418c812458200395651037642b2ce25172536b Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Thu, 7 Nov 2024 00:06:55 +0100 Subject: [PATCH 12/19] Add node count condition to _cluster/health --- elastic/logs/challenges/esql-ccs-snapshot.json | 6 ++++++ elastic/logs/track.json | 2 ++ 2 files changed, 8 insertions(+) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index a41bfcd97..45dc85667 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -52,6 +52,9 @@ "index": "_all", "request-params": { "wait_for_status": "{{cluster_health | default('green')}}", + {% if p_esql_ccs_local_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, + {% endif %} "wait_for_no_relocating_shards": "true" }, "retry-until-success": true @@ -147,6 +150,9 @@ "index": "_all", "request-params": { "wait_for_status": "{{wait_for_status | default('green')}}", + {% if p_esql_ccs_remote_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }}, + {% endif %} "wait_for_no_relocating_shards": "true" }, "retry-until-success": true diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 5b949f73d..80625824c 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -39,6 +39,8 @@ {% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_esql_ccs_skip_remote_clusters)) %} {% set p_esql_ccs_scaling_queries = (esql_ccs_scaling_queries | default(false)) %} {% set p_esql_ccs_extra_body = (esql_ccs_extra_body | default("")) %} +{% set p_esql_ccs_local_nodes_count = (esql_ccs_local_nodes_count | default(0)) %} +{% set p_esql_ccs_remote_nodes_count = (esql_ccs_remote_nodes_count | default(0)) %} {% set p_snapshot_repo_name = (snapshot_repo_name | default("logging")) %} {% set p_snapshot_repo_type = (snapshot_repo_type | default("s3")) %} From 8823f2d48f24459b66efa51c955d91f281ab9707 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Thu, 7 Nov 2024 15:37:10 +0100 Subject: [PATCH 13/19] Check cluster health at completion --- .../logs/challenges/esql-ccs-snapshot.json | 41 ++++++------------- elastic/logs/operations/esql.json | 32 +++++++++++++++ 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 45dc85667..a0b2165a5 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -47,18 +47,7 @@ { "name": "check-cluster-health-on-local-cluster", - "operation": { - "operation-type": "cluster-health", - "index": "_all", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - {% if p_esql_ccs_local_nodes_count > 0 %} - "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, - {% endif %} - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } + "operation": "check-cluster-health-on-local-cluster" }, {% if p_esql_ccs_include_local_snapshot %} @@ -143,20 +132,7 @@ }, { "name": "check-cluster-health-on-remote-clusters", - "operation": { - "operation-type": "multi-cluster-wrapper", - "base-operation-type": "cluster-health", - "ignore-clusters": ["default"], - "index": "_all", - "request-params": { - "wait_for_status": "{{wait_for_status | default('green')}}", - {% if p_esql_ccs_remote_nodes_count > 0 %} - "wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }}, - {% endif %} - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } + "operation": "check-cluster-health-on-remote-clusters" }, { "name": "setup-bi-directional-remote-clusters", @@ -182,7 +158,16 @@ } }, {% endif %}{# if not p_esql_ccs_skip_remote_clusters ... #} - - {% include "tasks/many-clusters-esql-ccs-queries.json" %} + {% include "tasks/many-clusters-esql-ccs-queries.json" %}, + { + "name": "check-cluster-health-on-local-cluster-at-completion", + "operation": "check-cluster-health-on-local-cluster" + } + {% if not p_esql_ccs_skip_remote_clusters %}, + { + "name": "check-cluster-health-on-remote-clusters-at-completion", + "operation": "check-cluster-health-on-remote-clusters" + } + {% endif %} ] } diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 0c8047f93..77147bcd8 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -333,4 +333,36 @@ "operation-type": "raw-request", "path": "/_cluster/stats?include_remotes=true", "method": "GET" + }, + { + "name": "check-cluster-health-on-local-cluster", + "operation": { + "operation-type": "cluster-health", + "index": "_all", + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + {% if p_esql_ccs_local_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, + {% endif %} + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } + }, + { + "name": "check-cluster-health-on-remote-clusters", + "operation": { + "operation-type": "multi-cluster-wrapper", + "base-operation-type": "cluster-health", + "ignore-clusters": ["default"], + "index": "_all", + "request-params": { + "wait_for_status": "{{wait_for_status | default('green')}}", + {% if p_esql_ccs_remote_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }}, + {% endif %} + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } } From 6f05f747768eb4a38cfe90d207e774fa10c60b7d Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Thu, 7 Nov 2024 16:01:26 +0100 Subject: [PATCH 14/19] Fix operation syntax --- elastic/logs/operations/esql.json | 38 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/elastic/logs/operations/esql.json b/elastic/logs/operations/esql.json index 77147bcd8..6dfdffeaa 100644 --- a/elastic/logs/operations/esql.json +++ b/elastic/logs/operations/esql.json @@ -336,33 +336,29 @@ }, { "name": "check-cluster-health-on-local-cluster", - "operation": { - "operation-type": "cluster-health", - "index": "_all", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - {% if p_esql_ccs_local_nodes_count > 0 %} - "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, - {% endif %} - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } + "operation-type": "cluster-health", + "index": "_all", + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + {% if p_esql_ccs_local_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, + {% endif %} + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true }, { "name": "check-cluster-health-on-remote-clusters", - "operation": { - "operation-type": "multi-cluster-wrapper", - "base-operation-type": "cluster-health", - "ignore-clusters": ["default"], - "index": "_all", - "request-params": { + "operation-type": "multi-cluster-wrapper", + "base-operation-type": "cluster-health", + "ignore-clusters": ["default"], + "index": "_all", + "request-params": { "wait_for_status": "{{wait_for_status | default('green')}}", {% if p_esql_ccs_remote_nodes_count > 0 %} "wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }}, {% endif %} "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } + }, + "retry-until-success": true } From 0f20b698ac97f8279aaed9cdb44e73c0fe3e2a09 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 12 Nov 2024 11:37:56 +0100 Subject: [PATCH 15/19] Add local snapshot restore option --- .../logs/challenges/esql-ccs-snapshot.json | 38 +++++++++++++++++++ elastic/logs/track.json | 1 + 2 files changed, 39 insertions(+) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index a0b2165a5..ce4a8fe5e 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -88,6 +88,44 @@ }, {% endif %} + {% if p_esql_ccs_include_local_snapshot_restore %} + {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} + { + "name": "register-snapshot-repository-on-local-cluster", + "operation": { + "operation-type": "create-snapshot-repository", + "repository": {{ p_snapshot_repo_name | tojson }}, + "body": { + "type": {{ p_snapshot_repo_type | tojson }}, + "settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}} + } + } + }, + { + "name": "recover-snapshot-on-local-cluster", + "operation": { + "operation-type": "restore-snapshot", + "repository": {{ p_snapshot_repo_name | tojson }}, + "snapshot": {{ p_snapshot_name | tojson }}, + "wait-for-completion": false, + "body": { + "indices": {{ p_restore_data_streams | tojson }}, + "ignore_unavailable": true, + "include_global_state": false, + "rename_pattern": "(.+)", + "rename_replacement": "$1{{ p_snapshot_rename_suffix }}" + } + } + }, + { + "name": "wait-for-snapshot-recovery-on-local-cluster", + "operation": { + "operation-type": "wait-for-recovery", + "index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }} + } + }, + {% endif %} + {% if not p_esql_ccs_skip_remote_clusters %} {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} { diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 80625824c..ee2a5ef9a 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -37,6 +37,7 @@ {% set p_esql_target_prefix = "remote*:" %} {% endif %} {% set p_esql_ccs_include_local_snapshot = (esql_ccs_include_local_snapshot | default(not p_esql_ccs_skip_remote_clusters)) %} +{% set p_esql_ccs_include_local_snapshot_restore = (esql_ccs_include_local_snapshot_restore | default(false)) %} {% set p_esql_ccs_scaling_queries = (esql_ccs_scaling_queries | default(false)) %} {% set p_esql_ccs_extra_body = (esql_ccs_extra_body | default("")) %} {% set p_esql_ccs_local_nodes_count = (esql_ccs_local_nodes_count | default(0)) %} From a7054483aa0e58d49c0db47836cabf7a77e9b15b Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 12 Nov 2024 12:56:15 +0100 Subject: [PATCH 16/19] Check local cluster health later --- elastic/logs/challenges/esql-ccs-snapshot.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index ce4a8fe5e..76d1b7b68 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -45,11 +45,6 @@ {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} {% endif %} - { - "name": "check-cluster-health-on-local-cluster", - "operation": "check-cluster-health-on-local-cluster" - }, - {% if p_esql_ccs_include_local_snapshot %} { "name": "register-snapshot-repository-on-local-cluster", @@ -126,6 +121,11 @@ }, {% endif %} + { + "name": "check-cluster-health-on-local-cluster", + "operation": "check-cluster-health-on-local-cluster" + }, + {% if not p_esql_ccs_skip_remote_clusters %} {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} { From 4238e7c0e50141be787d516c6c9a667a341dbd92 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 12 Nov 2024 13:00:39 +0100 Subject: [PATCH 17/19] Revert "Check local cluster health later" This reverts commit a7054483aa0e58d49c0db47836cabf7a77e9b15b. --- elastic/logs/challenges/esql-ccs-snapshot.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 76d1b7b68..ce4a8fe5e 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -45,6 +45,11 @@ {% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %} {% endif %} + { + "name": "check-cluster-health-on-local-cluster", + "operation": "check-cluster-health-on-local-cluster" + }, + {% if p_esql_ccs_include_local_snapshot %} { "name": "register-snapshot-repository-on-local-cluster", @@ -121,11 +126,6 @@ }, {% endif %} - { - "name": "check-cluster-health-on-local-cluster", - "operation": "check-cluster-health-on-local-cluster" - }, - {% if not p_esql_ccs_skip_remote_clusters %} {% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %} { From ed8051216ed6ab03add6b5f5c418e0e9e40d2b12 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 12 Nov 2024 13:02:19 +0100 Subject: [PATCH 18/19] Add cluster health check after local snapshot restore --- elastic/logs/challenges/esql-ccs-snapshot.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index ce4a8fe5e..07f14c307 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -124,6 +124,10 @@ "index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }} } }, + { + "name": "check-cluster-health-on-local-cluster-after-snapshot-restore", + "operation": "check-cluster-health-on-local-cluster" + }, {% endif %} {% if not p_esql_ccs_skip_remote_clusters %} From 5c729f18dbfc4be6d8072bcfb57349f5863324d8 Mon Sep 17 00:00:00 2001 From: Grzegorz Banasiak Date: Tue, 12 Nov 2024 13:12:36 +0100 Subject: [PATCH 19/19] At completion cluster checks without retries --- .../logs/challenges/esql-ccs-snapshot.json | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/elastic/logs/challenges/esql-ccs-snapshot.json b/elastic/logs/challenges/esql-ccs-snapshot.json index 07f14c307..9bb431fe2 100644 --- a/elastic/logs/challenges/esql-ccs-snapshot.json +++ b/elastic/logs/challenges/esql-ccs-snapshot.json @@ -200,15 +200,42 @@ } }, {% endif %}{# if not p_esql_ccs_skip_remote_clusters ... #} + {% include "tasks/many-clusters-esql-ccs-queries.json" %}, + + {# the following cluster health checks purposefully without retries #} { "name": "check-cluster-health-on-local-cluster-at-completion", - "operation": "check-cluster-health-on-local-cluster" + "operation": { + "operation-type": "cluster-health", + "index": "_all", + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + {% if p_esql_ccs_local_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_local_nodes_count }}, + {% endif %} + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": false + } } {% if not p_esql_ccs_skip_remote_clusters %}, { "name": "check-cluster-health-on-remote-clusters-at-completion", - "operation": "check-cluster-health-on-remote-clusters" + "operation": { + "operation-type": "multi-cluster-wrapper", + "base-operation-type": "cluster-health", + "ignore-clusters": ["default"], + "index": "_all", + "request-params": { + "wait_for_status": "{{wait_for_status | default('green')}}", + {% if p_esql_ccs_remote_nodes_count > 0 %} + "wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }}, + {% endif %} + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": false + } } {% endif %} ]