Store Template's mappings as bytes for disk serialization (elastic#78746)

probakowski · probakowski · commit 540397fed71a · 2021-10-20T00:19:02.000+02:00
This change the way we store mappings in `Template` during serialization
to disk - instead storing it as map we use byte array that we already
have. This avoids deserialization-serialization cycle during storing
cluster state on disk.
# Conflicts:
#	server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -79,7 +80,7 @@ private static void toXContent(GetComponentTemplatesResponse response, XContentB
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("component_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ private static void toXContent(GetComposableIndexTemplatesResponse response, XCo
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("index_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
@@ -23,20 +23,22 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.gateway.PersistedClusterStateService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -70,7 +72,8 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
         public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
             // Currently, two unknown top-level objects are present
             if (Metadata.Custom.class.isAssignableFrom(categoryClass)) {
-                if (DataStreamMetadata.TYPE.equals(name)) {
+                if (DataStreamMetadata.TYPE.equals(name) || ComposableIndexTemplateMetadata.TYPE.equals(name)
+                    || ComponentTemplateMetadata.TYPE.equals(name)) {
                     // DataStreamMetadata is used inside Metadata class for validation purposes and building the indicesLookup,
                     // therefor even es node commands need to be able to parse it.
                     return super.parseNamedObject(categoryClass, name, parser, context);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
@@ -129,7 +129,7 @@ public String toString() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(TEMPLATE.getPreferredName(), this.template);
+        builder.field(TEMPLATE.getPreferredName(), this.template, params);
         if (this.version != null) {
             builder.field(VERSION.getPreferredName(), this.version);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -98,7 +98,7 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
         for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue());
+            builder.field(template.getKey(), template.getValue(), params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
@@ -218,7 +218,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.stringListField(INDEX_PATTERNS.getPreferredName(), this.indexPatterns);
         if (this.template != null) {
-            builder.field(TEMPLATE.getPreferredName(), this.template);
+            builder.field(TEMPLATE.getPreferredName(), this.template, params);
         }
         if (this.componentTemplates != null) {
             builder.stringListField(COMPOSED_OF.getPreferredName(), this.componentTemplates);
@@ -233,7 +233,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(METADATA.getPreferredName(), metadata);
         }
         if (this.dataStreamTemplate != null) {
-            builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate);
+            builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate, params);
         }
         if (this.allowAutoCreate != null) {
             builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
@@ -99,7 +99,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(INDEX_TEMPLATE.getPreferredName());
         for (Map.Entry<String, ComposableIndexTemplate> template : indexTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue());
+            builder.field(template.getKey(), template.getValue(), params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -27,6 +28,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -47,8 +49,18 @@ public class Template extends AbstractDiffable<Template> implements ToXContentOb
 
     static {
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
-            new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            XContentParser.Token token = p.currentToken();
+            if (token == XContentParser.Token.VALUE_STRING) {
+                return new CompressedXContent(Base64.getDecoder().decode(p.text()));
+            } else if(token == XContentParser.Token.VALUE_EMBEDDED_OBJECT){
+                return new CompressedXContent(p.binaryValue());
+            } else if (token == XContentParser.Token.START_OBJECT){
+                return new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered())));
+            } else {
+                throw new IllegalArgumentException("Unexpected token: " + token);
+            }
+        }, MAPPINGS, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             Map<String, AliasMetadata> aliasMap = new HashMap<>();
             while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -160,11 +172,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
         }
         if (this.mappings != null) {
-            Map<String, Object> uncompressedMapping =
-                XContentHelper.convertToMap(this.mappings.uncompressed(), true, XContentType.JSON).v2();
-            if (uncompressedMapping.size() > 0) {
-                builder.field(MAPPINGS.getPreferredName());
-                builder.map(reduceMapping(uncompressedMapping));
+            String context = params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API);
+            boolean binary = params.paramAsBoolean("binary", false);
+            if (Metadata.CONTEXT_MODE_API.equals(context) || binary == false) {
+                Map<String, Object> uncompressedMapping =
+                    XContentHelper.convertToMap(this.mappings.uncompressed(), true, XContentType.JSON).v2();
+                if (uncompressedMapping.size() > 0) {
+                    builder.field(MAPPINGS.getPreferredName());
+                    builder.map(reduceMapping(uncompressedMapping));
+                }
+            } else {
+                builder.field(MAPPINGS.getPreferredName(), mappings.compressed());
             }
         }
         if (this.aliases != null) {

Original file line number	Diff line number	Diff line change
`@@ -129,7 +129,7 @@ public String toString() {`
`129`	`129`	`@Override`
`130`	`130`	`public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {`
`131`	`131`	`builder.startObject();`
`132`		`- builder.field(TEMPLATE.getPreferredName(), this.template);`
	`132`	`+ builder.field(TEMPLATE.getPreferredName(), this.template, params);`
`133`	`133`	`if (this.version != null) {`
`134`	`134`	`builder.field(VERSION.getPreferredName(), this.version);`
`135`	`135`	`}`
Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro`
`98`	`98`	`public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {`
`99`	`99`	`builder.startObject(COMPONENT_TEMPLATE.getPreferredName());`
`100`	`100`	`for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {`
`101`		`- builder.field(template.getKey(), template.getValue());`
	`101`	`+ builder.field(template.getKey(), template.getValue(), params);`
`102`	`102`	`}`
`103`	`103`	`builder.endObject();`
`104`	`104`	`return builder;`
Original file line number	Diff line number	Diff line change
`@@ -218,7 +218,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws`
`218`	`218`	`builder.startObject();`
`219`	`219`	`builder.stringListField(INDEX_PATTERNS.getPreferredName(), this.indexPatterns);`
`220`	`220`	`if (this.template != null) {`
`221`		`- builder.field(TEMPLATE.getPreferredName(), this.template);`
	`221`	`+ builder.field(TEMPLATE.getPreferredName(), this.template, params);`
`222`	`222`	`}`
`223`	`223`	`if (this.componentTemplates != null) {`
`224`	`224`	`builder.stringListField(COMPOSED_OF.getPreferredName(), this.componentTemplates);`
`@@ -233,7 +233,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws`
`233`	`233`	`builder.field(METADATA.getPreferredName(), metadata);`
`234`	`234`	`}`
`235`	`235`	`if (this.dataStreamTemplate != null) {`
`236`		`- builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate);`
	`236`	`+ builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate, params);`
`237`	`237`	`}`
`238`	`238`	`if (this.allowAutoCreate != null) {`
`239`	`239`	`builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);`
Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ public void writeTo(StreamOutput out) throws IOException {`
`99`	`99`	`public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {`
`100`	`100`	`builder.startObject(INDEX_TEMPLATE.getPreferredName());`
`101`	`101`	`for (Map.Entry<String, ComposableIndexTemplate> template : indexTemplates.entrySet()) {`
`102`		`- builder.field(template.getKey(), template.getValue());`
	`102`	`+ builder.field(template.getKey(), template.getValue(), params);`
`103`	`103`	`}`
`104`	`104`	`builder.endObject();`
`105`	`105`	`return builder;`