Skip to content

Commit 2245812

Browse files
dnhatndnhatn
committed
Merge branch 'master' into ccr
* master: Tests: Fix convert error tests to use fixed value (#32415) IndicesClusterStateService should replace an init. replica with an init. primary with the same aId (#32374) REST high-level client: parse back _ignored meta field (#32362) [CI] Mute DocumentSubsetReaderTests testSearch
2 parents aa3b6e0 + 34d006f commit 2245812

File tree

13 files changed

+190
-86
lines changed

13 files changed

+190
-86
lines changed

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,9 @@ public void testConvertIntLeadingZero() throws Exception {
6767
assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(10));
6868
}
6969

70-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
7170
public void testConvertIntHexError() {
7271
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
73-
String value = "0x" + randomAlphaOfLengthBetween(1, 10);
72+
String value = "0xnotanumber";
7473
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
7574
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.INTEGER, false);
7675
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
@@ -138,10 +137,9 @@ public void testConvertLongLeadingZero() throws Exception {
138137
assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
139138
}
140139

141-
@AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
142140
public void testConvertLongHexError() {
143141
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
144-
String value = "0x" + randomAlphaOfLengthBetween(1, 10);
142+
String value = "0xnotanumber";
145143
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, value);
146144
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
147145
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));

‎server/src/main/java/org/elasticsearch/index/get/GetResult.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.index.get;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.common.Strings;
2324
import org.elasticsearch.common.bytes.BytesReference;
2425
import org.elasticsearch.common.compress.CompressorFactory;
2526
import org.elasticsearch.common.document.DocumentField;
@@ -30,6 +31,7 @@
3031
import org.elasticsearch.common.xcontent.XContentBuilder;
3132
import org.elasticsearch.common.xcontent.XContentHelper;
3233
import org.elasticsearch.common.xcontent.XContentParser;
34+
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
3335
import org.elasticsearch.index.mapper.SourceFieldMapper;
3436
import org.elasticsearch.search.lookup.SourceLookup;
3537

@@ -225,10 +227,13 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
225227
}
226228
}
227229
}
228-
229230
for (DocumentField field : metaFields) {
230-
Object value = field.getValue();
231-
builder.field(field.getName(), value);
231+
// TODO: can we avoid having an exception here?
232+
if (field.getName().equals(IgnoredFieldMapper.NAME)) {
233+
builder.field(field.getName(), field.getValues());
234+
} else {
235+
builder.field(field.getName(), field.<Object>getValue());
236+
}
232237
}
233238

234239
builder.field(FOUND, exists);
@@ -316,7 +321,11 @@ public static GetResult fromXContentEmbedded(XContentParser parser, String index
316321
parser.skipChildren(); // skip potential inner objects for forward compatibility
317322
}
318323
} else if (token == XContentParser.Token.START_ARRAY) {
319-
parser.skipChildren(); // skip potential inner arrays for forward compatibility
324+
if (IgnoredFieldMapper.NAME.equals(currentFieldName)) {
325+
fields.put(currentFieldName, new DocumentField(currentFieldName, parser.list()));
326+
} else {
327+
parser.skipChildren(); // skip potential inner arrays for forward compatibility
328+
}
320329
}
321330
}
322331
return new GetResult(index, type, id, version, found, source, fields);
@@ -400,7 +409,12 @@ public boolean equals(Object o) {
400409

401410
@Override
402411
public int hashCode() {
403-
return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
412+
return Objects.hash(version, exists, index, type, id, fields, sourceAsMap());
413+
}
414+
415+
@Override
416+
public String toString() {
417+
return Strings.toString(this, true, true);
404418
}
405419
}
406420

‎server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,12 @@ private void removeShards(final ClusterState state) {
420420
// state may result in a new shard being initialized while having the same allocation id as the currently started shard.
421421
logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
422422
indexService.removeShard(shardId.id(), "removing shard (stale copy)");
423+
} else if (newShardRouting.primary() && currentRoutingEntry.primary() == false && newShardRouting.initializing()) {
424+
assert currentRoutingEntry.initializing() : currentRoutingEntry; // see above if clause
425+
// this can happen when cluster state batching batches activation of the shard, closing an index, reopening it
426+
// and assigning an initializing primary to this node
427+
logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting);
428+
indexService.removeShard(shardId.id(), "removing shard (stale copy)");
423429
}
424430
}
425431
}

‎server/src/main/java/org/elasticsearch/search/SearchHit.java‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -602,16 +602,24 @@ private static void declareMetaDataFields(ObjectParser<Map<String, Object>, Void
602602
for (String metadatafield : MapperService.getAllMetaFields()) {
603603
if (metadatafield.equals(Fields._ID) == false && metadatafield.equals(Fields._INDEX) == false
604604
&& metadatafield.equals(Fields._TYPE) == false) {
605-
parser.declareField((map, field) -> {
606-
@SuppressWarnings("unchecked")
607-
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
608-
v -> new HashMap<String, DocumentField>());
609-
fieldMap.put(field.getName(), field);
610-
}, (p, c) -> {
611-
List<Object> values = new ArrayList<>();
612-
values.add(parseFieldsValue(p));
613-
return new DocumentField(metadatafield, values);
614-
}, new ParseField(metadatafield), ValueType.VALUE);
605+
if (metadatafield.equals(IgnoredFieldMapper.NAME)) {
606+
parser.declareObjectArray((map, list) -> {
607+
@SuppressWarnings("unchecked")
608+
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
609+
v -> new HashMap<String, DocumentField>());
610+
DocumentField field = new DocumentField(metadatafield, list);
611+
fieldMap.put(field.getName(), field);
612+
}, (p, c) -> parseFieldsValue(p),
613+
new ParseField(metadatafield));
614+
} else {
615+
parser.declareField((map, field) -> {
616+
@SuppressWarnings("unchecked")
617+
Map<String, DocumentField> fieldMap = (Map<String, DocumentField>) map.computeIfAbsent(Fields.FIELDS,
618+
v -> new HashMap<String, DocumentField>());
619+
fieldMap.put(field.getName(), field);
620+
}, (p, c) -> new DocumentField(metadatafield, Collections.singletonList(parseFieldsValue(p))),
621+
new ParseField(metadatafield), ValueType.VALUE);
622+
}
615623
}
616624
}
617625
}
@@ -958,4 +966,9 @@ public int hashCode() {
958966
return Objects.hash(field, offset, child);
959967
}
960968
}
969+
970+
@Override
971+
public String toString() {
972+
return Strings.toString(this, true, true);
973+
}
961974
}

‎server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@
2727
import org.elasticsearch.cluster.metadata.MetaData;
2828
import org.elasticsearch.cluster.node.DiscoveryNode;
2929
import org.elasticsearch.cluster.node.DiscoveryNodes;
30+
import org.elasticsearch.cluster.routing.AllocationId;
3031
import org.elasticsearch.cluster.routing.IndexRoutingTable;
3132
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
3233
import org.elasticsearch.cluster.routing.RoutingTable;
3334
import org.elasticsearch.cluster.routing.RoutingTable.Builder;
35+
import org.elasticsearch.cluster.routing.ShardRouting;
3436
import org.elasticsearch.cluster.routing.ShardRoutingState;
3537
import org.elasticsearch.cluster.routing.TestShardRouting;
3638
import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -44,6 +46,7 @@
4446
import java.util.HashSet;
4547
import java.util.List;
4648
import java.util.Set;
49+
import java.util.stream.Collectors;
4750

4851
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
4952
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -93,7 +96,8 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard
9396
IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder()
9497
.put(SETTING_VERSION_CREATED, Version.CURRENT)
9598
.put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas)
96-
.put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
99+
.put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm)
100+
.build();
97101

98102
RoutingTable.Builder routing = new RoutingTable.Builder();
99103
routing.addAsNew(indexMetaData);
@@ -138,12 +142,19 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard
138142
TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState,
139143
unassignedInfo));
140144
}
145+
final IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build();
146+
147+
IndexMetaData.Builder indexMetaDataBuilder = new IndexMetaData.Builder(indexMetaData);
148+
indexMetaDataBuilder.putInSyncAllocationIds(0,
149+
indexShardRoutingTable.activeShards().stream().map(ShardRouting::allocationId).map(AllocationId::getId)
150+
.collect(Collectors.toSet())
151+
);
141152

142153
ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
143154
state.nodes(discoBuilder);
144-
state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded());
155+
state.metaData(MetaData.builder().put(indexMetaDataBuilder.build(), false).generateClusterUuidIfNeeded());
145156
state.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(indexMetaData.getIndex())
146-
.addIndexShard(indexShardRoutingBuilder.build())).build());
157+
.addIndexShard(indexShardRoutingTable)).build());
147158
return state.build();
148159
}
149160

@@ -272,21 +283,21 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index,
272283
state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());
273284
return state.build();
274285
}
275-
276-
286+
287+
277288
/**
278289
* Creates cluster state with several indexes, shards and replicas and all shards STARTED.
279290
*/
280291
public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) {
281292

282-
int numberOfDataNodes = numberOfReplicas + 1;
293+
int numberOfDataNodes = numberOfReplicas + 1;
283294
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
284295
for (int i = 0; i < numberOfDataNodes + 1; i++) {
285296
final DiscoveryNode node = newNode(i);
286297
discoBuilder = discoBuilder.add(node);
287298
}
288299
discoBuilder.localNodeId(newNode(0).getId());
289-
discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId());
300+
discoBuilder.masterNodeId(newNode(numberOfDataNodes + 1).getId());
290301
ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
291302
state.nodes(discoBuilder);
292303
Builder routingTableBuilder = RoutingTable.builder();
@@ -316,7 +327,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice
316327
state.metaData(metadataBuilder);
317328
state.routingTable(routingTableBuilder.build());
318329
return state.build();
319-
}
330+
}
320331

321332
/**
322333
* Creates cluster state with and index that has one shard and as many replicas as numberOfReplicas.

‎server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@
2626
import org.elasticsearch.common.xcontent.ToXContent;
2727
import org.elasticsearch.common.xcontent.XContentParser;
2828
import org.elasticsearch.common.xcontent.XContentType;
29-
import org.elasticsearch.index.mapper.RoutingFieldMapper;
29+
import org.elasticsearch.index.mapper.IdFieldMapper;
30+
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
31+
import org.elasticsearch.index.mapper.IndexFieldMapper;
32+
import org.elasticsearch.index.mapper.MapperService;
33+
import org.elasticsearch.index.mapper.TypeFieldMapper;
3034
import org.elasticsearch.test.ESTestCase;
3135
import org.elasticsearch.test.RandomObjects;
3236

@@ -98,14 +102,28 @@ private static DocumentField mutateDocumentField(DocumentField documentField) {
98102

99103
public static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xContentType) {
100104
if (randomBoolean()) {
101-
String fieldName = randomFrom(RoutingFieldMapper.NAME);
102-
DocumentField documentField = new DocumentField(fieldName, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
105+
String metaField = randomValueOtherThanMany(field -> field.equals(TypeFieldMapper.NAME)
106+
|| field.equals(IndexFieldMapper.NAME) || field.equals(IdFieldMapper.NAME),
107+
() -> randomFrom(MapperService.getAllMetaFields()));
108+
DocumentField documentField;
109+
if (metaField.equals(IgnoredFieldMapper.NAME)) {
110+
int numValues = randomIntBetween(1, 3);
111+
List<Object> ignoredFields = new ArrayList<>(numValues);
112+
for (int i = 0; i < numValues; i++) {
113+
ignoredFields.add(randomAlphaOfLengthBetween(3, 10));
114+
}
115+
documentField = new DocumentField(metaField, ignoredFields);
116+
} else {
117+
//meta fields are single value only, besides _ignored
118+
documentField = new DocumentField(metaField, Collections.singletonList(randomAlphaOfLengthBetween(3, 10)));
119+
}
103120
return Tuple.tuple(documentField, documentField);
121+
} else {
122+
String fieldName = randomAlphaOfLengthBetween(3, 10);
123+
Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
124+
DocumentField input = new DocumentField(fieldName, tuple.v1());
125+
DocumentField expected = new DocumentField(fieldName, tuple.v2());
126+
return Tuple.tuple(input, expected);
104127
}
105-
String fieldName = randomAlphaOfLengthBetween(3, 10);
106-
Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
107-
DocumentField input = new DocumentField(fieldName, tuple.v1());
108-
DocumentField expected = new DocumentField(fieldName, tuple.v2());
109-
return Tuple.tuple(input, expected);
110128
}
111129
}

‎server/src/test/java/org/elasticsearch/index/get/GetResultTests.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
9090
XContentType xContentType = randomFrom(XContentType.values());
9191
Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
9292
GetResult getResult = tuple.v1();
93-
9493
// We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
9594
// by the toXContentEmbedded method.
9695
GetResult expectedGetResult = new GetResult(null, null, null, -1,
@@ -106,7 +105,6 @@ public void testToAndFromXContentEmbedded() throws Exception {
106105
parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
107106
assertNull(parser.nextToken());
108107
}
109-
110108
assertEquals(expectedGetResult, parsedEmbeddedGetResult);
111109
//print the parsed object out and test that the output is the same as the original output
112110
BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
@@ -203,16 +201,17 @@ public static Tuple<GetResult, GetResult> randomGetResult(XContentType xContentT
203201
return Tuple.tuple(getResult, expectedGetResult);
204202
}
205203

206-
private static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
204+
public static Tuple<Map<String, DocumentField>,Map<String, DocumentField>> randomDocumentFields(XContentType xContentType) {
207205
int numFields = randomIntBetween(2, 10);
208206
Map<String, DocumentField> fields = new HashMap<>(numFields);
209207
Map<String, DocumentField> expectedFields = new HashMap<>(numFields);
210-
for (int i = 0; i < numFields; i++) {
208+
while (fields.size() < numFields) {
211209
Tuple<DocumentField, DocumentField> tuple = randomDocumentField(xContentType);
212210
DocumentField getField = tuple.v1();
213211
DocumentField expectedGetField = tuple.v2();
214-
fields.put(getField.getName(), getField);
215-
expectedFields.put(expectedGetField.getName(), expectedGetField);
212+
if (fields.putIfAbsent(getField.getName(), getField) == null) {
213+
assertNull(expectedFields.putIfAbsent(expectedGetField.getName(), expectedGetField));
214+
}
216215
}
217216
return Tuple.tuple(fields, expectedFields);
218217
}

‎server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ public void injectRandomFailures() {
7474
enableRandomFailures = randomBoolean();
7575
}
7676

77+
protected void disableRandomFailures() {
78+
enableRandomFailures = false;
79+
}
80+
7781
protected void failRandomly() {
7882
if (enableRandomFailures && rarely()) {
7983
throw new RuntimeException("dummy test failure");

0 commit comments

Comments
 (0)