Skip to content

Commit 982ff4e

Browse files
mgodwandk2k
mgodwan
authored and
dk2k
committed
Create Index using context (opensearch-project#15290)
* Create Index using context Signed-off-by: Mohit Godwani <[email protected]>
1 parent d2af34e commit 982ff4e

29 files changed

+934
-42
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3535
- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454))
3636
- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
3737
- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
38+
- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))
3839

3940
### Dependencies
4041
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

‎client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java

+25-4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import org.apache.lucene.util.CollectionUtil;
3636
import org.opensearch.cluster.metadata.AliasMetadata;
37+
import org.opensearch.cluster.metadata.Context;
3738
import org.opensearch.cluster.metadata.MappingMetadata;
3839
import org.opensearch.common.settings.Settings;
3940
import org.opensearch.core.xcontent.XContentParser;
@@ -61,6 +62,7 @@ public class GetIndexResponse {
6162
private Map<String, Settings> settings;
6263
private Map<String, Settings> defaultSettings;
6364
private Map<String, String> dataStreams;
65+
private Map<String, Context> contexts;
6466
private String[] indices;
6567

6668
GetIndexResponse(
@@ -69,7 +71,8 @@ public class GetIndexResponse {
6971
Map<String, List<AliasMetadata>> aliases,
7072
Map<String, Settings> settings,
7173
Map<String, Settings> defaultSettings,
72-
Map<String, String> dataStreams
74+
Map<String, String> dataStreams,
75+
Map<String, Context> contexts
7376
) {
7477
this.indices = indices;
7578
// to have deterministic order
@@ -89,6 +92,9 @@ public class GetIndexResponse {
8992
if (dataStreams != null) {
9093
this.dataStreams = dataStreams;
9194
}
95+
if (contexts != null) {
96+
this.contexts = contexts;
97+
}
9298
}
9399

94100
public String[] getIndices() {
@@ -123,6 +129,10 @@ public Map<String, String> getDataStreams() {
123129
return dataStreams;
124130
}
125131

132+
public Map<String, Context> contexts() {
133+
return contexts;
134+
}
135+
126136
/**
127137
* Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to
128138
* false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
@@ -167,6 +177,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
167177
Settings indexSettings = null;
168178
Settings indexDefaultSettings = null;
169179
String dataStream = null;
180+
Context context = null;
170181
// We start at START_OBJECT since fromXContent ensures that
171182
while (parser.nextToken() != Token.END_OBJECT) {
172183
ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser);
@@ -185,6 +196,9 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
185196
case "defaults":
186197
indexDefaultSettings = Settings.fromXContent(parser);
187198
break;
199+
case "context":
200+
context = Context.fromXContent(parser);
201+
break;
188202
default:
189203
parser.skipChildren();
190204
}
@@ -197,7 +211,7 @@ private static IndexEntry parseIndexEntry(XContentParser parser) throws IOExcept
197211
parser.skipChildren();
198212
}
199213
}
200-
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream);
214+
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream, context);
201215
}
202216

203217
// This is just an internal container to make stuff easier for returning
@@ -207,19 +221,22 @@ private static class IndexEntry {
207221
Settings indexSettings = Settings.EMPTY;
208222
Settings indexDefaultSettings = Settings.EMPTY;
209223
String dataStream;
224+
Context context;
210225

211226
IndexEntry(
212227
List<AliasMetadata> indexAliases,
213228
MappingMetadata indexMappings,
214229
Settings indexSettings,
215230
Settings indexDefaultSettings,
216-
String dataStream
231+
String dataStream,
232+
Context context
217233
) {
218234
if (indexAliases != null) this.indexAliases = indexAliases;
219235
if (indexMappings != null) this.indexMappings = indexMappings;
220236
if (indexSettings != null) this.indexSettings = indexSettings;
221237
if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings;
222238
if (dataStream != null) this.dataStream = dataStream;
239+
if (context != null) this.context = context;
223240
}
224241
}
225242

@@ -229,6 +246,7 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
229246
Map<String, Settings> settings = new HashMap<>();
230247
Map<String, Settings> defaultSettings = new HashMap<>();
231248
Map<String, String> dataStreams = new HashMap<>();
249+
Map<String, Context> contexts = new HashMap<>();
232250
List<String> indices = new ArrayList<>();
233251

234252
if (parser.currentToken() == null) {
@@ -254,12 +272,15 @@ public static GetIndexResponse fromXContent(XContentParser parser) throws IOExce
254272
if (indexEntry.dataStream != null) {
255273
dataStreams.put(indexName, indexEntry.dataStream);
256274
}
275+
if (indexEntry.context != null) {
276+
contexts.put(indexName, indexEntry.context);
277+
}
257278
} else if (parser.currentToken() == Token.START_ARRAY) {
258279
parser.skipChildren();
259280
} else {
260281
parser.nextToken();
261282
}
262283
}
263-
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams);
284+
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams, contexts);
264285
}
265286
}

‎client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.opensearch.client.AbstractResponseTestCase;
3737
import org.opensearch.client.GetAliasesResponseTests;
3838
import org.opensearch.cluster.metadata.AliasMetadata;
39+
import org.opensearch.cluster.metadata.Context;
3940
import org.opensearch.cluster.metadata.MappingMetadata;
4041
import org.opensearch.common.settings.IndexScopedSettings;
4142
import org.opensearch.common.settings.Settings;
@@ -66,6 +67,7 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
6667
final Map<String, Settings> settings = new HashMap<>();
6768
final Map<String, Settings> defaultSettings = new HashMap<>();
6869
final Map<String, String> dataStreams = new HashMap<>();
70+
final Map<String, Context> contexts = new HashMap<>();
6971
IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
7072
boolean includeDefaults = randomBoolean();
7173
for (String index : indices) {
@@ -90,14 +92,19 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
9092
if (randomBoolean()) {
9193
dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
9294
}
95+
96+
if (randomBoolean()) {
97+
contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
98+
}
9399
}
94100
return new org.opensearch.action.admin.indices.get.GetIndexResponse(
95101
indices,
96102
mappings,
97103
aliases,
98104
settings,
99105
defaultSettings,
100-
dataStreams
106+
dataStreams,
107+
null
101108
);
102109
}
103110

@@ -116,6 +123,7 @@ protected void assertInstances(
116123
assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
117124
assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
118125
assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
126+
assertEquals(serverTestInstance.contexts(), clientInstance.contexts());
119127
}
120128

121129
private static MappingMetadata createMappingsForIndex() {

‎server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java

+60
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,24 @@
4141
import org.opensearch.action.support.IndicesOptions;
4242
import org.opensearch.action.support.master.AcknowledgedResponse;
4343
import org.opensearch.cluster.ClusterState;
44+
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
45+
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
46+
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
47+
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
48+
import org.opensearch.cluster.metadata.Context;
4449
import org.opensearch.cluster.metadata.IndexMetadata;
4550
import org.opensearch.cluster.metadata.MappingMetadata;
4651
import org.opensearch.cluster.metadata.Metadata;
52+
import org.opensearch.cluster.service.ClusterService;
4753
import org.opensearch.common.settings.Settings;
4854
import org.opensearch.common.settings.SettingsException;
4955
import org.opensearch.common.unit.TimeValue;
5056
import org.opensearch.common.xcontent.XContentFactory;
5157
import org.opensearch.core.action.ActionListener;
58+
import org.opensearch.core.common.bytes.BytesReference;
5259
import org.opensearch.index.IndexNotFoundException;
5360
import org.opensearch.index.IndexService;
61+
import org.opensearch.index.IndexSettings;
5462
import org.opensearch.index.mapper.MapperParsingException;
5563
import org.opensearch.index.mapper.MapperService;
5664
import org.opensearch.index.query.RangeQueryBuilder;
@@ -59,7 +67,10 @@
5967
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
6068
import org.opensearch.test.OpenSearchIntegTestCase.Scope;
6169

70+
import java.nio.ByteBuffer;
71+
import java.nio.charset.StandardCharsets;
6272
import java.util.Map;
73+
import java.util.UUID;
6374
import java.util.concurrent.CountDownLatch;
6475
import java.util.concurrent.atomic.AtomicInteger;
6576
import java.util.function.BiFunction;
@@ -430,4 +441,53 @@ public void testCreateIndexWithNullReplicaCountPickUpClusterReplica() {
430441
);
431442
}
432443
}
444+
445+
public void testCreateIndexWithContextSettingsAndTemplate() throws Exception {
446+
int numReplicas = 1;
447+
String indexName = "test-idx-1";
448+
Settings settings = Settings.builder()
449+
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
450+
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), (String) null)
451+
.build();
452+
Context context = new Context("test");
453+
454+
String templateContent = "{\n"
455+
+ " \"template\": {\n"
456+
+ " \"settings\": {\n"
457+
+ " \"merge.policy\": \"log_byte_size\"\n"
458+
+ " }\n"
459+
+ " },\n"
460+
+ " \"_meta\": {\n"
461+
+ " \"_type\": \"@abc_template\",\n"
462+
+ " \"_version\": 1\n"
463+
+ " },\n"
464+
+ " \"version\": 1\n"
465+
+ "}\n";
466+
467+
ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
468+
internalCluster().clusterManagerClient(),
469+
() -> internalCluster().getInstance(ClusterService.class).state()
470+
);
471+
loader.loadTemplate(
472+
new SystemTemplate(
473+
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
474+
SystemTemplateMetadata.fromComponentTemplateInfo("test", 1L),
475+
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
476+
)
477+
);
478+
479+
assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settings).setContext(context).get());
480+
481+
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, internalCluster().getClusterManagerName());
482+
483+
for (IndexService indexService : indicesService) {
484+
assertEquals(indexName, indexService.index().getName());
485+
assertEquals(
486+
numReplicas,
487+
(int) indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null)
488+
);
489+
assertEquals(context, indexService.getMetadata().context());
490+
assertEquals("log_byte_size", indexService.getMetadata().getSettings().get(IndexSettings.INDEX_MERGE_POLICY.getKey()));
491+
}
492+
}
433493
}

‎server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java

+62
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,19 @@
3535
import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
3636
import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
3737
import org.opensearch.cluster.ClusterState;
38+
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
39+
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
40+
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
41+
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
42+
import org.opensearch.cluster.metadata.Context;
3843
import org.opensearch.cluster.metadata.IndexMetadata;
3944
import org.opensearch.cluster.service.ClusterService;
4045
import org.opensearch.common.Priority;
4146
import org.opensearch.common.settings.Setting;
4247
import org.opensearch.common.settings.Settings;
4348
import org.opensearch.common.settings.SettingsException;
4449
import org.opensearch.common.unit.TimeValue;
50+
import org.opensearch.core.common.bytes.BytesReference;
4551
import org.opensearch.index.IndexModule;
4652
import org.opensearch.index.IndexService;
4753
import org.opensearch.index.VersionType;
@@ -51,10 +57,14 @@
5157
import org.opensearch.test.OpenSearchIntegTestCase;
5258
import org.opensearch.threadpool.ThreadPool;
5359

60+
import java.io.IOException;
61+
import java.nio.ByteBuffer;
62+
import java.nio.charset.StandardCharsets;
5463
import java.util.Arrays;
5564
import java.util.Collection;
5665
import java.util.Collections;
5766
import java.util.List;
67+
import java.util.UUID;
5868

5969
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA;
6070
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ;
@@ -99,6 +109,58 @@ public void testInvalidDynamicUpdate() {
99109
assertNotEquals(indexMetadata.getSettings().get("index.dummy"), "invalid dynamic value");
100110
}
101111

112+
public void testDynamicUpdateWithContextSettingOverlap() throws IOException {
113+
String templateContent = "{\n"
114+
+ " \"template\": {\n"
115+
+ " \"settings\": {\n"
116+
+ " \"index.merge.policy\": \"log_byte_size\"\n"
117+
+ " }\n"
118+
+ " },\n"
119+
+ " \"_meta\": {\n"
120+
+ " \"_type\": \"@abc_template\",\n"
121+
+ " \"_version\": 1\n"
122+
+ " },\n"
123+
+ " \"version\": 1\n"
124+
+ "}\n";
125+
126+
ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
127+
internalCluster().clusterManagerClient(),
128+
() -> internalCluster().getInstance(ClusterService.class).state()
129+
);
130+
loader.loadTemplate(
131+
new SystemTemplate(
132+
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
133+
SystemTemplateMetadata.fromComponentTemplateInfo("testcontext", 1L),
134+
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
135+
)
136+
);
137+
138+
createIndex("test", new Context("testcontext"));
139+
140+
IllegalArgumentException validationException = expectThrows(
141+
IllegalArgumentException.class,
142+
() -> client().admin()
143+
.indices()
144+
.prepareUpdateSettings("test")
145+
.setSettings(Settings.builder().put("index.merge.policy", "tiered"))
146+
.execute()
147+
.actionGet()
148+
);
149+
assertTrue(
150+
validationException.getMessage()
151+
.contains("Cannot apply context template as user provide settings have overlap with the included context template")
152+
);
153+
154+
assertAcked(
155+
client().admin()
156+
.indices()
157+
.prepareUpdateSettings("test")
158+
.setSettings(Settings.builder().put("index.refresh_interval", "60s"))
159+
.execute()
160+
.actionGet()
161+
);
162+
}
163+
102164
@Override
103165
protected Collection<Class<? extends Plugin>> nodePlugins() {
104166
return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class);

‎server/src/main/java/org/opensearch/OpenSearchServerException.java

+8
Original file line numberDiff line numberDiff line change
@@ -1201,6 +1201,14 @@ public static void registerExceptions() {
12011201
V_2_13_0
12021202
)
12031203
);
1204+
registerExceptionHandle(
1205+
new OpenSearchExceptionHandle(
1206+
org.opensearch.indices.InvalidIndexContextException.class,
1207+
org.opensearch.indices.InvalidIndexContextException::new,
1208+
174,
1209+
V_3_0_0
1210+
)
1211+
);
12041212
registerExceptionHandle(
12051213
new OpenSearchExceptionHandle(
12061214
org.opensearch.cluster.block.IndexCreateBlockException.class,

0 commit comments

Comments
 (0)