Make the tsdb track compatible with Elasticsearch 7.17

tsdb/README.md

@@ -195,6 +195,7 @@ This track allows to overwrite the following parameters using `--track-params`:
 * `number_of_shards` (default: 1)
 * `refresh_interval` (default not defined)
 * `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use.
+* `include_source_mode` (default: true): used to explicitly exclude `index.mapping.source.mode` setting when running with Elasticsearch v7.
 * `source_mode` (default: synthetic): Should the `_source` be `stored` to disk exactly as sent (the Elasticsearch default outside of TSDB mode), thrown away (`disabled`), or reconstructed on the fly (`synthetic`)
 * `index_mode` (default: time_series): Whether to make a standard index (`standard`) or time series index (`time_series`)
 * `codec` (default: default): The codec to use compressing the index. `default` uses more space and less cpu. `best_compression` uses less space and more cpu.

tsdb/challenges/default.json

@@ -126,7 +126,7 @@
           "clients": {{ search_clients | default(1) | int }},
           "iterations": 100
         }
-        {% if index_mode | default('time_series') is equalto 'time_series' and skip_running_tsdb_aggs is not defined %}
+        {% if p_index_mode == "time_series" and skip_running_tsdb_aggs is not defined %}
         ,{
           "operation": "date-histo-memory-usage-hour",
           "warmup-iterations": 50,
@@ -150,16 +150,17 @@
           "warmup-iterations": 50,
           "clients": {{ search_clients | default(1) | int }},
           "iterations": 100
-        }
+        },
         {% endif %}
-        ,{
+        {
           "operation": "esql-fetch-500",
           "warmup-iterations": 50,
           "clients": {{ search_clients | default(1) | int }},
           "iterations": 100
         }
       ]
-    },
+    }
+    {% if p_enable_downsample_challenge %},
     {
       "name": "downsample",
       "description": "Indexes the whole document corpus and executes a few downsampling operations using different interval values",
@@ -281,7 +282,9 @@
           "iterations": 100
         }
       ]
-    },
+    }
+    {% endif %}
+    {% if p_enable_low_latency_challenge %},
     {
       "name": "low-latency",
       "description": "Indexes the whole document corpus and executes low-latency queries. Assertions might result in race failure.",
@@ -383,7 +386,7 @@
           }
         }
         {%- endif -%}{# serverless-post-ingest-sleep-marker-end #}
-        {% if index_mode | default('time_series') is equalto 'standard' %}
+        {% if p_index_mode != "time_series" and p_index_mode != "logsdb" %}
         ,{
           "operation": "search_by_doc_id",
           "warmup-iterations": 50,
@@ -407,3 +410,4 @@
         {%- endif %}
       ]
     }
+    {% endif %}

tsdb/index-template.json

@@ -1,6 +1,7 @@
 {% import "rally.helpers" as rally with context %}
 {# TODO By default, source_mode should be "synthetic" if index_mode == "time_series" else "stored" #}
 {% set p_source_mode = (source_mode | default("synthetic")) %}
+{% set p_include_source_mode = (include_source_mode | default(true)) %}
 {
   "index_patterns": ["k8s*"],
   "data_stream": {},
@@ -16,8 +17,10 @@
         "refresh_interval": "{{refresh_interval}}",
         {% endif %}
         "codec": "{{codec | default('default')}}",
-        "mode": "{{index_mode | default('time_series')}}",
-        {% if index_mode | default('time_series') is equalto 'time_series' %}
+        {% if p_index_mode == "time_series" or p_index_mode == "logsdb" %}
+        "mode": "{{p_index_mode}}",
+        {% endif %}
+        {% if p_index_mode == "time_series" %}
         "routing_path": [
           "metricset.name",
           "kubernetes.container.name",
@@ -35,8 +38,10 @@
           {% if synthetic_source_keep %}
           "synthetic_source_keep": "{{synthetic_source_keep}}",
           {% endif %}
-          "total_fields.limit": 10000,
+          "total_fields.limit": 10000
+          {% if p_include_source_mode %},
           "source.mode": {{ p_source_mode | tojson }}
+          {% endif %}
         }
       }
     },

tsdb/index.json

@@ -1,6 +1,7 @@
 {% import "rally.helpers" as rally with context %}
 {# TODO By default, source_mode should be "synthetic" if index_mode == "time_series" else "stored" #}
 {% set p_source_mode = (source_mode | default("synthetic")) %}
+{% set p_include_source_mode = (include_source_mode | default(true)) %}
 {
   "settings": {
     "index": {
@@ -13,11 +14,13 @@
       {% endif %}
       {%- endif -%}{# non-serverless-index-settings-marker-end #}
       "codec": "{{codec | default('default')}}",
-      "mode": "{{index_mode | default('time_series')}}",
+      {% if p_index_mode == "time_series" or p_index_mode == "logsdb" %}
+      "mode": "{{p_index_mode}}",
+      {% endif %}
       {% if refresh_interval %}
       "refresh_interval": "{{refresh_interval}}",
       {% endif %}
-      {% if index_mode | default('time_series') is equalto 'time_series' %}
+      {% if p_index_mode == "time_series" %}
       "routing_path": [
         "metricset.name",
         "kubernetes.container.name",
@@ -35,8 +38,10 @@
         {% if synthetic_source_keep %}
         "synthetic_source_keep": "{{synthetic_source_keep}}",
         {% endif %}
-        "total_fields.limit": 10000,
+        "total_fields.limit": 10000
+        {% if p_include_source_mode %},
         "source.mode": {{ p_source_mode | tojson }}
+        {% endif %}
       }
     }
   },

tsdb/operations/default.json

@@ -1,4 +1,6 @@
 {% set p_document_ids = (document_ids | default(["Rpgg6gSYETjE4x8VrIAPNQ==", "AVdIG3TktHA+a3U2gW29QA==", "DeiNU57QgAml0UMiPg7hWA==", "3o6jhoyitLP3RPjSAEMRgQ=="])) %}
+{% set p_include_time_series_aggs = (include_time_series_aggs | default(true)) %}
+
 {
   "name": "delete-doc-id-pipeline",
   "operation-type": "raw-request",

tsdb/track.json

@@ -7,6 +7,10 @@
 {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
 {% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %}
 
+{% set p_index_mode = (index_mode | default("time_series")) %}
+{% set p_enable_downsample_challenge = (enable_downsample_challenge | default(false)) %}
+{% set p_enable_low_latency_challenge = (enable_low_latency_challenge | default(false)) %}
+
 {
   "version": 2,
   "description": "metricbeat information for elastic-app k8s cluster",