From 44aea10f9b17e4470c5b93c656f056887386ee13 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 28 Jan 2025 17:37:58 +0200 Subject: [PATCH 1/5] Set source mode through an index setting --- cohere_vector/index-vectors-only-mapping.json | 6 +----- .../templates/component/track-shared-logsdb-mode.json | 11 +++-------- .../templates/component/track-shared-logsdb-mode.json | 10 +++------- http_logs/index.json | 6 +++--- .../templates/metrics-kubernetes.pod@custom.json | 4 +--- .../templates/metrics-kubernetes.pod@package.json | 4 +--- msmarco-v2-vector/index-vectors-only-mapping.json | 4 +--- nyc_taxis/index.json | 7 ++----- random_vector/index-mapping.json | 6 ++---- tsdb/index-template.json | 6 ++---- tsdb/index.json | 6 ++---- 11 files changed, 21 insertions(+), 49 deletions(-) diff --git a/cohere_vector/index-vectors-only-mapping.json b/cohere_vector/index-vectors-only-mapping.json index bfdd00250..b8e586582 100644 --- a/cohere_vector/index-vectors-only-mapping.json +++ b/cohere_vector/index-vectors-only-mapping.json @@ -4,17 +4,13 @@ {% if preload_pagecache %} "index.store.preload": [ "vec", "vex", "vem"], {% endif %} + "index.mapping.source.mode": "synthetic", "index.number_of_shards": {{number_of_shards | default(1)}}, "index.number_of_replicas": {{number_of_replicas | default(0)}} {%- endif -%}{# non-serverless-index-settings-marker-end #} }, "mappings": { "dynamic": false, - {%- if build_flavor != "serverless" -%} - "_source": { - "mode": "synthetic" - }, - {%- endif -%} "properties": { "emb": { "type": "dense_vector", diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index 7831005d7..f4eeea48f 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -4,6 +4,9 @@ {% if index_mode %} "index": { "mode": {{ index_mode | tojson }} + {% if source_mode %} + ,"mapping.source.mode": {{ source_mode | tojson }} + {% endif %} {% if use_synthetic_source_recovery %} ,"recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}} {% endif %} @@ -11,14 +14,6 @@ ,"mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson }} {% endif %} } - {% endif %} - }, - "mappings": { - {% if source_mode %} - "_source": { - "mode": {{ source_mode | tojson }} - } - {% endif %} } } } diff --git a/elastic/security/templates/component/track-shared-logsdb-mode.json b/elastic/security/templates/component/track-shared-logsdb-mode.json index fc644f403..1b590707f 100644 --- a/elastic/security/templates/component/track-shared-logsdb-mode.json +++ b/elastic/security/templates/component/track-shared-logsdb-mode.json @@ -4,6 +4,9 @@ {% if index_mode %} "index": { "mode": {{ index_mode | tojson }}, + {% if source_mode %} + "mapping.source.mode": {{ source_mode | tojson }}, + {% endif %} {% if use_synthetic_source_recovery %} "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, {% endif %} @@ -15,13 +18,6 @@ "sort.missing": ["_first", "_last"] } {% endif %} - }, - "mappings": { - {% if source_mode %} - "_source": { - "mode": {{ source_mode | tojson }} - } - {% endif %} } } } diff --git a/http_logs/index.json b/http_logs/index.json index 1e696e5a9..513041867 100644 --- a/http_logs/index.json +++ b/http_logs/index.json @@ -13,6 +13,9 @@ "index.sort.field": ["@timestamp"], "index.sort.order":["desc"] {%- endif %} + {% if source_mode %} + ,"mapping.source.mode": {{ source_mode | tojson }} + {% endif %} }, "mappings": { "dynamic": "strict", @@ -20,9 +23,6 @@ {%- else %} "_source": { "enabled": {{ source_enabled | default(true) | tojson }} - {% if source_mode %} - "mode": {{ source_mode | tojson }}, - {% endif %} }, {%- endif %} "properties": { diff --git a/k8s_metrics/templates/metrics-kubernetes.pod@custom.json b/k8s_metrics/templates/metrics-kubernetes.pod@custom.json index 6ac8d40be..8acd860a0 100644 --- a/k8s_metrics/templates/metrics-kubernetes.pod@custom.json +++ b/k8s_metrics/templates/metrics-kubernetes.pod@custom.json @@ -14,6 +14,7 @@ {%- endif %} "codec": "best_compression", "mapping": { + "source.mode": "synthetic", "total_fields": { "limit": "10000" } @@ -70,9 +71,6 @@ } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "dynamic_templates": [ { "container.labels": { diff --git a/k8s_metrics/templates/metrics-kubernetes.pod@package.json b/k8s_metrics/templates/metrics-kubernetes.pod@package.json index b87bbb1a0..e3e4cd780 100644 --- a/k8s_metrics/templates/metrics-kubernetes.pod@package.json +++ b/k8s_metrics/templates/metrics-kubernetes.pod@package.json @@ -7,6 +7,7 @@ "index": { "codec": "best_compression", "mapping": { + "source.mode": "synthetic", "total_fields": { "limit": "10000" } @@ -63,9 +64,6 @@ } }, "mappings": { - "_source": { - "mode": "synthetic" - }, "dynamic_templates": [ { "container.labels": { diff --git a/msmarco-v2-vector/index-vectors-only-mapping.json b/msmarco-v2-vector/index-vectors-only-mapping.json index de5227bab..735f6f18c 100644 --- a/msmarco-v2-vector/index-vectors-only-mapping.json +++ b/msmarco-v2-vector/index-vectors-only-mapping.json @@ -8,6 +8,7 @@ {% if preload_pagecache %} "store.preload": [ "vec", "vex", "vem"], {% endif %} + "mapping.source.mode": "synthetic", "number_of_shards": {{number_of_shards | default(1)}}, "number_of_replicas": {{number_of_replicas | default(0)}} {% if aggressive_merge_policy %}, @@ -24,9 +25,6 @@ }, "mappings": { "dynamic": false, - "_source": { - "mode": "synthetic" - }, "properties": { "docid": { "type": "keyword" diff --git a/nyc_taxis/index.json b/nyc_taxis/index.json index 00910de6f..8051cb6dd 100644 --- a/nyc_taxis/index.json +++ b/nyc_taxis/index.json @@ -7,14 +7,11 @@ "index.number_of_shards": {{number_of_shards | default(1)}}, "index.number_of_replicas": {{number_of_replicas | default(0)}}, {% endif %} - "index.requests.cache.enable": false + "index.requests.cache.enable": false, {%- endif -%}{# non-serverless-index-settings-marker-end #} + "index.mapping.source.mode": {{ source_mode | default("stored") | tojson }} }, "mappings": { - "_source": { - "mode": {{ source_mode | default("stored") | tojson }} - - }, "properties": { "surcharge": { "scaling_factor": 100, diff --git a/random_vector/index-mapping.json b/random_vector/index-mapping.json index 316ff7e07..2dcbf5fb7 100644 --- a/random_vector/index-mapping.json +++ b/random_vector/index-mapping.json @@ -7,16 +7,14 @@ {%- endif -%}{# non-serverless-index-settings-marker-end #} "sort": { "field": "partition_id" - } + }, + "mapping.source.mode": "synthetic" } }, "mappings": { "_routing": { "required": true }, - "_source": { - "mode": "synthetic" - }, "properties": { "partition_id": { "type": "keyword" diff --git a/tsdb/index-template.json b/tsdb/index-template.json index 37d76a1e8..1e10b98e3 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -35,7 +35,8 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000 + "total_fields.limit": 10000, + "source.mode": {{ p_source_mode | tojson }} } } }, @@ -44,9 +45,6 @@ "beat": "metricbeat", "version": "7.6.2" }, - "_source": { - "mode": {{ p_source_mode | tojson }} - }, "dynamic_templates": [ { "labels": { diff --git a/tsdb/index.json b/tsdb/index.json index d5fe6feae..cd0f6cc8d 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -35,7 +35,8 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000 + "total_fields.limit": 10000, + "source.mode": {{ p_source_mode | tojson }} } } }, @@ -44,9 +45,6 @@ "beat": "metricbeat", "version": "7.6.2" }, - "_source": { - "mode": {{ p_source_mode | tojson }} - }, "dynamic_templates": [ { "labels": { From ac2e3226223de91c7bec5d5886eeba7239826e5a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 28 Jan 2025 17:45:08 +0200 Subject: [PATCH 2/5] fix --- elastic/logs/templates/component/track-shared-logsdb-mode.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index f4eeea48f..552b064c4 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -14,6 +14,6 @@ ,"mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson }} {% endif %} } - } + {% endif %} } } From e48905066b3fb2a957b06cf80a7c63bee19015f3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 28 Jan 2025 17:54:07 +0200 Subject: [PATCH 3/5] Update track-shared-logsdb-mode.json --- elastic/logs/templates/component/track-shared-logsdb-mode.json | 1 + 1 file changed, 1 insertion(+) diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index 552b064c4..41326b5d1 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -15,5 +15,6 @@ {% endif %} } {% endif %} + } } } From 982372f1569932adc2dcee8fcdcd9e5e844c3ac1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 28 Jan 2025 18:53:03 +0200 Subject: [PATCH 4/5] fix --- nyc_taxis/index.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyc_taxis/index.json b/nyc_taxis/index.json index 8051cb6dd..13705f63e 100644 --- a/nyc_taxis/index.json +++ b/nyc_taxis/index.json @@ -8,8 +8,8 @@ "index.number_of_replicas": {{number_of_replicas | default(0)}}, {% endif %} "index.requests.cache.enable": false, - {%- endif -%}{# non-serverless-index-settings-marker-end #} "index.mapping.source.mode": {{ source_mode | default("stored") | tojson }} + {%- endif -%}{# non-serverless-index-settings-marker-end #} }, "mappings": { "properties": { From e5c1fa795c646ed8f156e0d4f6820b91aaa5ec8c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 28 Jan 2025 19:33:11 +0200 Subject: [PATCH 5/5] fix --- tsdb/index-template.json | 4 ++-- tsdb/index.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tsdb/index-template.json b/tsdb/index-template.json index 1e10b98e3..de1c85a4b 100644 --- a/tsdb/index-template.json +++ b/tsdb/index-template.json @@ -8,6 +8,7 @@ "settings": { "index": { {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%} + "mapping.source.mode": {{ p_source_mode | tojson }}, "number_of_shards": {{number_of_shards | default(1)}}, "number_of_replicas": {{number_of_replicas | default(0)}}, "requests.cache.enable": false, @@ -35,8 +36,7 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000, - "source.mode": {{ p_source_mode | tojson }} + "total_fields.limit": 10000 } } }, diff --git a/tsdb/index.json b/tsdb/index.json index cd0f6cc8d..940047bbc 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -5,6 +5,7 @@ "settings": { "index": { {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%} + "mapping.source.mode": {{ p_source_mode | tojson }}, "number_of_shards": {{number_of_shards | default(1)}}, "number_of_replicas": {{number_of_replicas | default(0)}}, "requests.cache.enable": false, @@ -35,8 +36,7 @@ {% if synthetic_source_keep %} "synthetic_source_keep": "{{synthetic_source_keep}}", {% endif %} - "total_fields.limit": 10000, - "source.mode": {{ p_source_mode | tojson }} + "total_fields.limit": 10000 } } },