Skip to content

Add new ESQL CCS challenge in Logs #701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gbanasiak merged 7 commits into from
Oct 30, 2024
Merged

Add new ESQL CCS challenge in Logs #701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8cfc399
Initial esql-ccs
gbanasiak Oct 16, 2024
7f35ebf
Modify cluster health task name
gbanasiak Oct 17, 2024
3bb1fc8
Wrap query cache enable/disable
gbanasiak Oct 18, 2024
52484c0
Make wrapper conditional
gbanasiak Oct 18, 2024
335896c
Ignore default cluster, and switch to default restore
gbanasiak Oct 18, 2024
5f36b86
Merge branch 'master' into esql-ccs
elasticmachine Oct 29, 2024
13f190c
Add comment
gbanasiak Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 167 additions & 0 deletions elastic/logs/challenges/esql-ccs-snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
{% import "rally.helpers" as rally %}
{
"name": "esql-ccs-snapshot",
"description": "Indexes logs to the local cluster, and runs ESQL queries either locally or across remote clusters via CCS. Data replicated via snapshot/restore.",
"schedule": [
{% if not p_skip_esql_ccs_remote_clusters %}
{% include "tasks/many-clusters-setup.json" %},
{% endif %}
{% include "tasks/index-setup.json" %},
{
"name": "bulk-index",
"operation": {
"operation-type": "raw-bulk",
"param-source": "processed-source",
"time-format": "milliseconds",
"profile": "fixed_interval",
"bulk-size": {{ p_bulk_size }},
"detailed-results": true
},
"clients": {{ p_bulk_indexing_clients }}{% if p_throttle_indexing %},
"ignore-response-error-level": "{{error_level | default('non-fatal')}}",
"schedule": "timestamp-throttler",
"max-delay-secs": 1
{% endif %}
},
{
"name": "force-merge-data-streams",
"operation": {
"operation-type": "force-merge",
"mode": "polling",
"max-num-segments": 1,
"request-timeout": 60
}
},
{% if p_skip_esql_ccs_remote_clusters %}
{
"name": "check-cluster-health-local",
"operation": {
"operation-type": "cluster-health",
"index": "_all",
"request-params": {
"wait_for_status": "{{cluster_health | default('green')}}",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{% else %}{# if p_skip_esql_ccs_remote_clusters ... #}
{# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #}
{% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %}
{# randomize base path to avoid clashes when running concurrent benchmarks #}
{% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %}
{
"name": "register-snapshot-repository-on-local-cluster",
"operation": {
"operation-type": "create-snapshot-repository",
"repository": {{ p_snapshot_repo_name | tojson }},
"body": {
"type": {{ p_snapshot_repo_type | tojson }},
"settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}}
}
}
},
{% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %}
{
"name": "register-snapshot-repository-on-remote-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "create-snapshot-repository",
"ignore-clusters": ["default"],
"repository": {{ p_snapshot_repo_name | tojson }},
"body": {
"type": {{ p_snapshot_repo_type | tojson }},
"settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}}
}
}
},
{
"name": "create-snapshot-{{ p_snapshot_name }}",
"operation": {
"operation-type": "create-snapshot",
"repository": {{ p_snapshot_repo_name | tojson }},
"snapshot": {{ p_snapshot_name | tojson }},
"wait-for-completion": false,
"body": {
"indices": {{ p_restore_data_streams | tojson }},
"ignore_unavailable": true,
"include_global_state": false,
"metadata": {{ p_snapshot_metadata | tojson }}
},
"include-in-reporting": true
}
},
{
"name": "wait-for-snapshot-{{ p_snapshot_name }}",
"operation": {
"operation-type": "wait-for-snapshot-create",
"snapshot": {{ p_snapshot_name | tojson }},
"repository": {{ p_snapshot_repo_name | tojson }}
}
},
{
"name": "recover-snapshot-{{ p_snapshot_name }}-on-remote-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "restore-snapshot",
"ignore-clusters": ["default"],
"repository": {{ p_snapshot_repo_name | tojson }},
"snapshot": {{ p_snapshot_name | tojson }},
"wait-for-completion": false,
"body": {
"indices": {{ p_restore_data_streams | tojson }},
"ignore_unavailable": true,
"include_global_state": false,
"rename_pattern": "(.+)",
"rename_replacement": "$1{{ p_snapshot_rename_suffix }}"
}
}
},
{
"name": "wait-for-snapshot-recovery-{{ p_snapshot_name }}-on-all-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "wait-for-recovery",
"index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }}
}
},
{
"name": "check-cluster-health-all-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "cluster-health",
"index": "_all",
"request-params": {
"wait_for_status": "{{wait_for_status | default('green')}}",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{
"name": "setup-bi-directional-remote-clusters",
"operation": {
"operation-type": "configure-remote-clusters",
"local-cluster": "default"
}
},
{
"name": "reset-recovery-settings-on-all-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "raw-request",
"method": "PUT",
"path": "/_cluster/settings",
"body": {
"persistent": {
"indices.recovery.max_bytes_per_sec": null,
"cluster.routing.allocation.node_concurrent_recoveries": null,
"cluster.routing.allocation.node_initial_primaries_recoveries": null
}
}
}
},
{% endif %}{# if p_skip_esql_ccs_remote_clusters ... #}
{% include "tasks/many-clusters-esql-ccs-queries.json" %}
]
}
76 changes: 52 additions & 24 deletions elastic/logs/operations/esql.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
{
"name": "enable_query_cache",
"include-in-reporting": false,
"operation-type": "raw-request",
{% if p_skip_esql_ccs_remote_clusters %}
"operation-type": "raw-request",
{% else %}
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "raw-request",
"ignore-clusters": ["default"],
{% endif %}
"method": "PUT",
"path": "/logs-*/_settings",
"body": {
Expand All @@ -11,42 +17,64 @@
{
"name": "disable_query_cache",
"include-in-reporting": false,
"operation-type": "raw-request",
{% if p_skip_esql_ccs_remote_clusters %}
"operation-type": "raw-request",
{% else %}
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "raw-request",
"ignore-clusters": ["default"],
{% endif %}
"method": "PUT",
"path": "/logs-*/_settings",
"body": {
"index.requests.cache.enable": false
}
},
{
"name": "restore_query_cache_default",
"include-in-reporting": false,
{% if p_skip_esql_ccs_remote_clusters %}
"operation-type": "raw-request",
{% else %}
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "raw-request",
"ignore-clusters": ["default"],
{% endif %}
"method": "PUT",
"path": "/logs-*/_settings",
"body": {
"index.requests.cache.enable": null
}
},
{
"name": "esql_basic_count_group_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) | SORT count DESC | LIMIT 20"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) | SORT count DESC | LIMIT 20"
},
{
"name": "esql_basic_count_group_1",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version | SORT count DESC | LIMIT 20"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version | SORT count DESC | LIMIT 20"
},
{
"name": "esql_basic_count_group_2",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type | SORT count DESC | LIMIT 20"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type | SORT count DESC | LIMIT 20"
},
{
"name": "esql_basic_count_group_3",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname | SORT count DESC | LIMIT 20"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname | SORT count DESC | LIMIT 20"
},
{
"name": "esql_basic_count_group_4",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname, agent.id | SORT count DESC | LIMIT 20"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname, agent.id | SORT count DESC | LIMIT 20"
},
{
"name": "search_basic_count_group_0",
"operation-type": "search",
"index": "logs-*",
"index": "{{p_esql_target_prefix}}logs-*",
"body": {
"size": 0,
"aggs": {
Expand All @@ -61,7 +89,7 @@
{
"name": "search_basic_count_group_1",
"operation-type": "search",
"index": "logs-*",
"index": "{{p_esql_target_prefix}}logs-*",
"body": {
"size": 0,
"aggs": {
Expand All @@ -77,7 +105,7 @@
{
"name": "search_basic_count_group_2",
"operation-type": "search",
"index": "logs-*",
"index": "{{p_esql_target_prefix}}logs-*",
"body":{
"size": 0,
"aggs": {
Expand All @@ -96,7 +124,7 @@
{
"name": "search_basic_count_group_3",
"operation-type": "search",
"index": "logs-*",
"index": "{{p_esql_target_prefix}}logs-*",
"body":{
"size": 0,
"aggs": {
Expand All @@ -116,7 +144,7 @@
{
"name": "search_basic_count_group_4",
"operation-type": "search",
"index": "logs-*",
"index": "{{p_esql_target_prefix}}logs-*",
"body":{
"size": 0,
"aggs": {
Expand All @@ -138,65 +166,65 @@
"name": "esql_time_range_and_date_histogram_two_groups_pre_filter",
"description": "Based on observability queries for average CPU over date histogram",
"operation-type": "esql",
"query": "FROM logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time AND http.response.body.bytes IS NOT NULL | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | KEEP data_stream.dataset, bucket, min, avg, max"
"query": "FROM {{p_esql_target_prefix}}logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time AND http.response.body.bytes IS NOT NULL | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | KEEP data_stream.dataset, bucket, min, avg, max"
},
{
"name": "esql_time_range_and_date_histogram_two_groups_post_filter",
"description": "Based on observability queries for average CPU over date histogram",
"operation-type": "esql",
"query": "FROM logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | WHERE min IS NOT NULL | KEEP data_stream.dataset, bucket, min, avg, max"
"query": "FROM {{p_esql_target_prefix}}logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | WHERE min IS NOT NULL | KEEP data_stream.dataset, bucket, min, avg, max"
},
{
"name": "esql_dissect_duration_and_stats",
"description": "Based on observability queries for duration average",
"operation-type": "esql",
"query": "FROM logs-postgres* | DISSECT message \"duration: %{query_duration} ms\" | EVAL query_duration_num = TO_DOUBLE(query_duration) | STATS avg_duration = AVG(query_duration_num)"
"query": "FROM {{p_esql_target_prefix}}logs-postgres* | DISSECT message \"duration: %{query_duration} ms\" | EVAL query_duration_num = TO_DOUBLE(query_duration) | STATS avg_duration = AVG(query_duration_num)"
},
{
"name": "esql_basic_count_group_0_limit_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) | SORT count DESC | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) | SORT count DESC | LIMIT 0"
},
{
"name": "esql_basic_count_group_1_limit_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version | SORT count DESC | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version | SORT count DESC | LIMIT 0"
},
{
"name": "esql_basic_count_group_2_limit_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type | SORT count DESC | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type | SORT count DESC | LIMIT 0"
},
{
"name": "esql_basic_count_group_3_limit_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname | SORT count DESC | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname | SORT count DESC | LIMIT 0"
},
{
"name": "esql_basic_count_group_4_limit_0",
"operation-type": "esql",
"query": "FROM logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname, agent.id | SORT count DESC | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | STATS count=count(*) BY agent.version, agent.type, agent.hostname, agent.id | SORT count DESC | LIMIT 0"
},
{
"name": "esql_time_range_and_date_histogram_two_groups_pre_filter_limit_0",
"description": "Based on observability queries for average CPU over date histogram",
"operation-type": "esql",
"query": "FROM logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time AND http.response.body.bytes IS NOT NULL | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | KEEP data_stream.dataset, bucket, min, avg, max | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time AND http.response.body.bytes IS NOT NULL | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | KEEP data_stream.dataset, bucket, min, avg, max | LIMIT 0"
},
{
"name": "esql_time_range_and_date_histogram_two_groups_post_filter_limit_0",
"description": "Based on observability queries for average CPU over date histogram",
"operation-type": "esql",
"query": "FROM logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | WHERE min IS NOT NULL | KEEP data_stream.dataset, bucket, min, avg, max | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-* | EVAL start_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_start_date }}\"), end_time = DATE_PARSE(\"yyyy-MM-dd\",\"{{ bulk_end_date }}\") | WHERE @timestamp >= start_time AND @timestamp <= end_time | EVAL bucket = DATE_TRUNC(1 hour, @timestamp) | STATS avg=AVG(http.response.body.bytes), min=MIN(http.response.body.bytes), max=MAX(http.response.body.bytes) BY data_stream.dataset, bucket | WHERE min IS NOT NULL | KEEP data_stream.dataset, bucket, min, avg, max | LIMIT 0"
},
{
"name": "esql_dissect_duration_and_stats_limit_0",
"description": "Based on observability queries for duration average",
"operation-type": "esql",
"query": "FROM logs-postgres* | DISSECT message \"duration: %{query_duration} ms\" | EVAL query_duration_num = TO_DOUBLE(query_duration) | STATS avg_duration = AVG(query_duration_num) | LIMIT 0"
"query": "FROM {{p_esql_target_prefix}}logs-postgres* | DISSECT message \"duration: %{query_duration} ms\" | EVAL query_duration_num = TO_DOUBLE(query_duration) | STATS avg_duration = AVG(query_duration_num) | LIMIT 0"
},
{
"name": "esql_from_all_limit_50",
"operation-type": "esql",
"query": "FROM * | LIMIT 50"
"query": "FROM {{p_esql_target_prefix}}* | LIMIT 50"
}
Loading