From 897d032c40d1a59a906d2b971bd86a76eb358967 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 30 Sep 2024 17:02:51 +0200 Subject: [PATCH 1/2] Add synthetic_source_keep param to tsdb and http_logs tracks --- http_logs/README.md | 1 + http_logs/index.json | 4 +++- tsdb/README.md | 1 + tsdb/index-template.json | 7 ++++++- tsdb/index.json | 7 ++++++- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/http_logs/README.md b/http_logs/README.md index d103ce57d..2f0123f56 100644 --- a/http_logs/README.md +++ b/http_logs/README.md @@ -51,6 +51,7 @@ node pipeline to run. Valid options are `'baseline'` (default), `'grok'` and `' * `post_ingest_sleep_duration` (default: 30): Sleep duration in seconds. * `enable_logsdb` (default: false) Determines whether the logsdb index mode gets used. If set then index sorting is configured to only use `@timestamp` field and the `source_enabled` parameter will have no effect. * `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. +* `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting. ### License diff --git a/http_logs/index.json b/http_logs/index.json index eb6b5db35..50db90888 100644 --- a/http_logs/index.json +++ b/http_logs/index.json @@ -7,7 +7,9 @@ {%- endif -%}{# non-serverless-index-settings-marker-end #} {%- if enable_logsdb %}, "index.mode": "logsdb", - "index.mapping.synthetic_source_keep": "arrays", + {% if synthetic_source_keep %} + "index.mapping.synthetic_source_keep": "{{synthetic_source_keep}}", + {% endif %} "index.sort.field": ["@timestamp"], "index.sort.order":["desc"] {%- endif %} diff --git a/tsdb/README.md b/tsdb/README.md index f9c66161f..696003259 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -203,6 +203,7 @@ This track allows to overwrite the following parameters using `--track-params`: * `post_ingest_sleep` (default: false): Whether to pause after ingest and prior to subsequent operations. * `post_ingest_sleep_duration` (default: 30): Sleep duration in seconds. * `document_ids`: documentd IDs to use for search, get and mget apis in the `low-latency` challenge. If empty, a default set of 4 values is used. +* `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting. ### License diff --git a/tsdb/index-template.json b/tsdb/index-template.json index d86cef559..37d76a1e8 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -31,7 +31,12 @@ "end_time": "2021-05-01T00:00:00.000Z" }, {% endif %} - "mapping.total_fields.limit": 10000 + "mapping": { + {% if synthetic_source_keep %} + "synthetic_source_keep": "{{synthetic_source_keep}}", + {% endif %} + "total_fields.limit": 10000 + } } }, "mappings": { diff --git a/tsdb/index.json b/tsdb/index.json index 1119eaaba..6ff1c0041 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -28,7 +28,12 @@ "end_time": "9999-01-01T00:00:00.000Z" }, {% endif %} - "mapping.total_fields.limit": 10000 + "mapping": { + {% if synthetic_source_keep %} + "synthetic_source_keep": "{{synthetic_source_keep}}", + {% endif %} + "total_fields.limit": 10000 + } } }, "mappings": { From b202a941ab0f70f6e2b4fe1d9ef667ed89669f74 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 2 Oct 2024 08:48:51 +0200 Subject: [PATCH 2/2] Fix small mistake in elastic/logs template when no synthetic_source_keep is specified. --- .../logs/templates/component/track-shared-logsdb-mode.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index dce96a167..05754531b 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -3,9 +3,9 @@ "settings": { {% if index_mode %} "index": { - "mode": {{ index_mode | tojson }}, + "mode": {{ index_mode | tojson }} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} - "mapping": { + ,"mapping": { "synthetic_source_keep": "{{ synthetic_source_keep }}" } {% endif %}