From 12d394eaaa5db785ebf40563caed3a2366a35102 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 13:01:42 +0100 Subject: [PATCH 01/14] feature: make tsdb append_no_conflict challenge compatible with v7 --- tsdb/challenges/default.json | 4 +++- tsdb/index-template.json | 4 +++- tsdb/track.json | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 5a6f54305..4417070b6 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -152,12 +152,14 @@ "iterations": 100 } {% endif %} - ,{ + {%- if p_include_esql_queries %}, + { "operation": "esql-fetch-500", "warmup-iterations": 50, "clients": {{ search_clients | default(1) | int }}, "iterations": 100 } + {% endif %} ] }, { diff --git a/tsdb/index-template.json b/tsdb/index-template.json index 1e10b98e3..22b0cbbc9 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -35,8 +35,10 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000, + "total_fields.limit": 10000 + {% if p_source_mode == "synthetic" or p_source_mode == "stored" %}, "source.mode": {{ p_source_mode | tojson }} + {% endif %} } } }, diff --git a/tsdb/track.json b/tsdb/track.json index 7c3e8749b..ed5a50b66 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -7,6 +7,8 @@ {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %} {% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} +{% set p_include_esql_queries = (include_esql_queries | default(build_flavor != "serverless")) %} + { "version": 2, "description": "metricbeat information for elastic-app k8s cluster", From df9e3a0d7b24c6faf461df09ecfa375a2ed41dc0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 14:10:17 +0100 Subject: [PATCH 02/14] fix: gate index.mode usage --- tsdb/index-template.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tsdb/index-template.json b/tsdb/index-template.json index 22b0cbbc9..40fbd2686 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -16,7 +16,9 @@ "refresh_interval": "{{refresh_interval}}", {% endif %} "codec": "{{codec | default('default')}}", + {% if index_mode == "time_series" or "index_mode" == "logsdb" %} "mode": "{{index_mode | default('time_series')}}", + {% endif %} {% if index_mode | default('time_series') is equalto 'time_series' %} "routing_path": [ "metricset.name", From c4bd463dc11a00909ab513e5010ab86c54b89d36 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 14:21:20 +0100 Subject: [PATCH 03/14] fix: gate index mode and set default to time_series --- tsdb/challenges/default.json | 4 ++-- tsdb/index-template.json | 6 +++--- tsdb/index.json | 6 ++++-- tsdb/track.json | 1 + 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 4417070b6..7f49f6b66 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -126,7 +126,7 @@ "clients": {{ search_clients | default(1) | int }}, "iterations": 100 } - {% if index_mode | default('time_series') is equalto 'time_series' and skip_running_tsdb_aggs is not defined %} + {% if p_index_mode == "time_series" and skip_running_tsdb_aggs is not defined %} ,{ "operation": "date-histo-memory-usage-hour", "warmup-iterations": 50, @@ -385,7 +385,7 @@ } } {%- endif -%}{# serverless-post-ingest-sleep-marker-end #} - {% if index_mode | default('time_series') is equalto 'standard' %} + {% if p_index_mode != "time_series" and p_index_mode != "logsdb" %} ,{ "operation": "search_by_doc_id", "warmup-iterations": 50, diff --git a/tsdb/index-template.json b/tsdb/index-template.json index 40fbd2686..b1154ceea 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -16,10 +16,10 @@ "refresh_interval": "{{refresh_interval}}", {% endif %} "codec": "{{codec | default('default')}}", - {% if index_mode == "time_series" or "index_mode" == "logsdb" %} - "mode": "{{index_mode | default('time_series')}}", + {% if p_index_mode == "time_series" or p_index_mode == "logsdb" %} + "mode": "{{p_index_mode}}", {% endif %} - {% if index_mode | default('time_series') is equalto 'time_series' %} + {% if p_index_mode == "time_series" %} "routing_path": [ "metricset.name", "kubernetes.container.name", diff --git a/tsdb/index.json b/tsdb/index.json index cd0f6cc8d..f4ada9b22 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -13,11 +13,13 @@ {% endif %} {%- endif -%}{# non-serverless-index-settings-marker-end #} "codec": "{{codec | default('default')}}", - "mode": "{{index_mode | default('time_series')}}", + {% if p_index_mode == "time_series" or p_index_mode == "logsdb" %} + "mode": "{{p_index_mode}}", + {% endif %} {% if refresh_interval %} "refresh_interval": "{{refresh_interval}}", {% endif %} - {% if index_mode | default('time_series') is equalto 'time_series' %} + {% if p_index_mode == "time_series" %} "routing_path": [ "metricset.name", "kubernetes.container.name", diff --git a/tsdb/track.json b/tsdb/track.json index ed5a50b66..b8a01fd3a 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -8,6 +8,7 @@ {% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} {% set p_include_esql_queries = (include_esql_queries | default(build_flavor != "serverless")) %} +{% set p_index_mode = (index_mode | default("time_series")) %} { "version": 2, From fdfc20e991cbe2d6dec8a2739685ea93ef88f814 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 14:23:38 +0100 Subject: [PATCH 04/14] fix: p_include_esql_queries true --- tsdb/track.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/track.json b/tsdb/track.json index b8a01fd3a..330464def 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -7,7 +7,7 @@ {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %} {% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} -{% set p_include_esql_queries = (include_esql_queries | default(build_flavor != "serverless")) %} +{% set p_include_esql_queries = (include_esql_queries | default(true)) %} {% set p_index_mode = (index_mode | default("time_series")) %} { From 02edd432d802702749954f1ca0b66c289a56f6be Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 14:59:30 +0100 Subject: [PATCH 05/14] fix: gate another index.mapping.source.mode usage --- tsdb/index.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tsdb/index.json b/tsdb/index.json index f4ada9b22..df803cf6f 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -37,8 +37,10 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000, + "total_fields.limit": 10000 + {% if p_source_mode == "synthetic" || p_source_mode == "stored" %}, "source.mode": {{ p_source_mode | tojson }} + {% endif %} } } }, From b10afce920a448aad4bf96870d595a445019a8a8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 15:34:33 +0100 Subject: [PATCH 06/14] fix: or statement --- tsdb/index.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/index.json b/tsdb/index.json index df803cf6f..5f74e8fc0 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -38,7 +38,7 @@ "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} "total_fields.limit": 10000 - {% if p_source_mode == "synthetic" || p_source_mode == "stored" %}, + {% if p_source_mode == "synthetic" or p_source_mode == "stored" %}, "source.mode": {{ p_source_mode | tojson }} {% endif %} } From 8096661a1c54f59786c614a23195bd1c59318db1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 16:23:14 +0100 Subject: [PATCH 07/14] fix: use a flag to icnlude/exclude index.mapping.source.mode --- tsdb/README.md | 1 + tsdb/index-template.json | 3 ++- tsdb/index.json | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tsdb/README.md b/tsdb/README.md index f088ae28b..0994cd8c0 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -195,6 +195,7 @@ This track allows to overwrite the following parameters using `--track-params`: * `number_of_shards` (default: 1) * `refresh_interval` (default not defined) * `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. +* `include_source_mode` (default: true): used to explicitly exclude `index.mapping.source.mode` setting when running with Elasticsearch v7. * `source_mode` (default: synthetic): Should the `_source` be `stored` to disk exactly as sent (the Elasticsearch default outside of TSDB mode), thrown away (`disabled`), or reconstructed on the fly (`synthetic`) * `index_mode` (default: time_series): Whether to make a standard index (`standard`) or time series index (`time_series`) * `codec` (default: default): The codec to use compressing the index. `default` uses more space and less cpu. `best_compression` uses less space and more cpu. diff --git a/tsdb/index-template.json b/tsdb/index-template.json index b1154ceea..c00918c23 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -1,6 +1,7 @@ {% import "rally.helpers" as rally with context %} {# TODO By default, source_mode should be "synthetic" if index_mode == "time_series" else "stored" #} {% set p_source_mode = (source_mode | default("synthetic")) %} +{% set p_include_source_mode = (include_source_mode | default(true)) %} { "index_patterns": ["k8s*"], "data_stream": {}, @@ -38,7 +39,7 @@ "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} "total_fields.limit": 10000 - {% if p_source_mode == "synthetic" or p_source_mode == "stored" %}, + {% if p_include_source_mode %}, "source.mode": {{ p_source_mode | tojson }} {% endif %} } diff --git a/tsdb/index.json b/tsdb/index.json index 5f74e8fc0..d22ecb39f 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -1,6 +1,7 @@ {% import "rally.helpers" as rally with context %} {# TODO By default, source_mode should be "synthetic" if index_mode == "time_series" else "stored" #} {% set p_source_mode = (source_mode | default("synthetic")) %} +{% set p_include_source_mode = (include_source_mode | default(true))) %} { "settings": { "index": { @@ -38,7 +39,7 @@ "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} "total_fields.limit": 10000 - {% if p_source_mode == "synthetic" or p_source_mode == "stored" %}, + {% if p_include_source_mode %}, "source.mode": {{ p_source_mode | tojson }} {% endif %} } From 880c009bf2959236442a4018201f033a7a519c8f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 16:36:09 +0100 Subject: [PATCH 08/14] fix: remove unnecessary parenthesis --- tsdb/index.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/index.json b/tsdb/index.json index d22ecb39f..ad38c0dc2 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -1,7 +1,7 @@ {% import "rally.helpers" as rally with context %} {# TODO By default, source_mode should be "synthetic" if index_mode == "time_series" else "stored" #} {% set p_source_mode = (source_mode | default("synthetic")) %} -{% set p_include_source_mode = (include_source_mode | default(true))) %} +{% set p_include_source_mode = (include_source_mode | default(true)) %} { "settings": { "index": { From c47bd01565d1c14a88a3212bb49e8ff6882727c5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 18:12:04 +0100 Subject: [PATCH 09/14] fix: gate execution of downsampling and low_latency challenges --- tsdb/challenges/default.json | 8 ++++++-- tsdb/operations/default.json | 2 ++ tsdb/track.json | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 7f49f6b66..1d1162105 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -161,7 +161,8 @@ } {% endif %} ] - }, + } + {% p_enable_downsamplig_challenge %}, { "name": "downsample", "description": "Indexes the whole document corpus and executes a few downsampling operations using different interval values", @@ -283,7 +284,9 @@ "iterations": 100 } ] - }, + } + {% endif %} + {% if p_enable_low_latency_challenge %}, { "name": "low-latency", "description": "Indexes the whole document corpus and executes low-latency queries. Assertions might result in race failure.", @@ -409,3 +412,4 @@ {%- endif %} ] } + {% endif %} diff --git a/tsdb/operations/default.json b/tsdb/operations/default.json index cecc30305..3bf97d609 100644 --- a/tsdb/operations/default.json +++ b/tsdb/operations/default.json @@ -1,4 +1,6 @@ {% set p_document_ids = (document_ids | default(["Rpgg6gSYETjE4x8VrIAPNQ==", "AVdIG3TktHA+a3U2gW29QA==", "DeiNU57QgAml0UMiPg7hWA==", "3o6jhoyitLP3RPjSAEMRgQ=="])) %} +{% set p_include_time_series_aggs = (include_time_series_aggs | default(true)) %} + { "name": "delete-doc-id-pipeline", "operation-type": "raw-request", diff --git a/tsdb/track.json b/tsdb/track.json index 330464def..c5a5edaa4 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -9,6 +9,8 @@ {% set p_include_esql_queries = (include_esql_queries | default(true)) %} {% set p_index_mode = (index_mode | default("time_series")) %} +{% set p_enable_downsample_challenge = (enable_downsample_challenge | default(true)) %} +{% set p_enable_low_latency_challenge = (enable_low_latency_challenge | default(true)) %} { "version": 2, From 2fffb015c629aac15c6c2455ff2da29f1ef65737 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 19:01:46 +0100 Subject: [PATCH 10/14] fix: variable name --- tsdb/challenges/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 1d1162105..914fcac3f 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -162,7 +162,7 @@ {% endif %} ] } - {% p_enable_downsamplig_challenge %}, + {% p_enable_downsample_challenge %}, { "name": "downsample", "description": "Indexes the whole document corpus and executes a few downsampling operations using different interval values", From 2e926e63719e4c74bcd8b617a67882e6cc0aebe6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Feb 2025 19:17:35 +0100 Subject: [PATCH 11/14] fix: missing if --- tsdb/challenges/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 914fcac3f..e9ec7ce9f 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -162,7 +162,7 @@ {% endif %} ] } - {% p_enable_downsample_challenge %}, + {% if p_enable_downsample_challenge %}, { "name": "downsample", "description": "Indexes the whole document corpus and executes a few downsampling operations using different interval values", From 5370e3dd52ba72ef8df61373fdd3191e875e8fd7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 14 Feb 2025 10:01:25 +0100 Subject: [PATCH 12/14] fix: default false --- tsdb/track.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsdb/track.json b/tsdb/track.json index c5a5edaa4..ad86ff761 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -9,8 +9,8 @@ {% set p_include_esql_queries = (include_esql_queries | default(true)) %} {% set p_index_mode = (index_mode | default("time_series")) %} -{% set p_enable_downsample_challenge = (enable_downsample_challenge | default(true)) %} -{% set p_enable_low_latency_challenge = (enable_low_latency_challenge | default(true)) %} +{% set p_enable_downsample_challenge = (enable_downsample_challenge | default(false)) %} +{% set p_enable_low_latency_challenge = (enable_low_latency_challenge | default(false)) %} { "version": 2, From df8057233a8013940e1aee1573cc1c7b4ebe33e2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 14 Feb 2025 11:45:32 +0100 Subject: [PATCH 13/14] fix: remove unnecessary track parameter --- tsdb/challenges/default.json | 2 -- tsdb/track.json | 1 - 2 files changed, 3 deletions(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index e9ec7ce9f..446010754 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -152,14 +152,12 @@ "iterations": 100 } {% endif %} - {%- if p_include_esql_queries %}, { "operation": "esql-fetch-500", "warmup-iterations": 50, "clients": {{ search_clients | default(1) | int }}, "iterations": 100 } - {% endif %} ] } {% if p_enable_downsample_challenge %}, diff --git a/tsdb/track.json b/tsdb/track.json index ad86ff761..a871071ad 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -7,7 +7,6 @@ {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %} {% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} -{% set p_include_esql_queries = (include_esql_queries | default(true)) %} {% set p_index_mode = (index_mode | default("time_series")) %} {% set p_enable_downsample_challenge = (enable_downsample_challenge | default(false)) %} {% set p_enable_low_latency_challenge = (enable_low_latency_challenge | default(false)) %} From be81ee51104684ad6cd745290996c08091d3170b Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 14 Feb 2025 12:06:11 +0100 Subject: [PATCH 14/14] fix: missing comma --- tsdb/challenges/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsdb/challenges/default.json b/tsdb/challenges/default.json index 446010754..c69e8c7ea 100644 --- a/tsdb/challenges/default.json +++ b/tsdb/challenges/default.json @@ -150,7 +150,7 @@ "warmup-iterations": 50, "clients": {{ search_clients | default(1) | int }}, "iterations": 100 - } + }, {% endif %} { "operation": "esql-fetch-500",