Skip to content

Strengthen field-caps responses wire test #84596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 15, 2022
Merged

Strengthen field-caps responses wire test #84596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
101feb5
Strengthen field-caps responses wire test
dnhatn Mar 3, 2022
aa01585
Merge branch 'master' into wire_test_field_caps
dnhatn Mar 8, 2022
8657d34
Two smaller tests
dnhatn Mar 8, 2022
ec0d856
Feedback
dnhatn Mar 9, 2022
fb2b489
Merge branch 'master' into wire_test_field_caps
dnhatn Mar 9, 2022
275e327
spotless
dnhatn Mar 9, 2022
d2253d5
Feedback
dnhatn Mar 15, 2022
f7b2181
Merge remote-tracking branch 'elastic/master' into wire_test_field_caps
dnhatn Mar 15, 2022
f7c9e5c
Shuffle
dnhatn Mar 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 52 additions & 86 deletions ...src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,107 +8,73 @@

package org.elasticsearch.action.fieldcaps;

import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.index.mapper.TimeSeriesParams;
import org.elasticsearch.test.ESTestCase;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.hamcrest.Matchers.equalTo;

public class FieldCapabilitiesIndexResponseTests extends ESTestCase {

public void testShareResponsesUsingMappingHash() throws Exception {
final Supplier<Map<String, IndexFieldCapabilities>> randomFieldCaps = () -> {
final Map<String, IndexFieldCapabilities> fieldCaps = new HashMap<>();
final List<String> fields = randomList(1, 5, () -> randomAlphaOfLength(5));
for (String field : fields) {
final IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
field,
randomAlphaOfLengthBetween(5, 20),
randomBoolean(),
randomBoolean(),
randomBoolean(),
false,
null,
Map.of()
);
fieldCaps.put(field, fieldCap);
}
return fieldCaps;
static Map<String, IndexFieldCapabilities> randomFieldCaps() {
final Map<String, IndexFieldCapabilities> fieldCaps = new HashMap<>();
final Map<String, String> meta = switch (randomInt(2)) {
case 0 -> Map.of();
case 1 -> Map.of("key", "value");
default -> Map.of("key1", "value1", "key2", "value2");
};
final List<FieldCapabilitiesIndexResponse> inList = new ArrayList<>();
int numGroups = randomIntBetween(0, 20);
Map<String, List<String>> mappingHashToIndices = new HashMap<>();
for (int i = 0; i < numGroups; i++) {
String groupName = "group_" + i;
String hashing = UUIDs.randomBase64UUID();
List<String> indices = IntStream.range(0, randomIntBetween(1, 5)).mapToObj(n -> groupName + "_" + n).toList();
mappingHashToIndices.put(hashing, indices);
Map<String, IndexFieldCapabilities> fieldCaps = randomFieldCaps.get();
for (String index : indices) {
inList.add(new FieldCapabilitiesIndexResponse(index, hashing, fieldCaps, true));
}
final TimeSeriesParams.MetricType metricType = randomBoolean() ? null : randomFrom(TimeSeriesParams.MetricType.values());
final List<String> fields = randomList(1, 5, () -> randomAlphaOfLength(5));
for (String field : fields) {
final IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
field,
randomAlphaOfLengthBetween(5, 20),
randomBoolean(),
randomBoolean(),
randomBoolean(),
randomBoolean(),
metricType,
meta
);
fieldCaps.put(field, fieldCap);
}
int numUngroups = randomIntBetween(0, 5);
for (int i = 0; i < numUngroups; i++) {
String index = "ungrouped_" + i;
final String hashing;
final boolean canMatch;
Map<String, IndexFieldCapabilities> fieldCaps = Map.of();
if (randomBoolean()) {
canMatch = false;
hashing = UUIDs.randomBase64UUID();
} else {
canMatch = randomBoolean();
hashing = null;
if (canMatch) {
fieldCaps = randomFieldCaps.get();
}
}
inList.add(new FieldCapabilitiesIndexResponse(index, hashing, fieldCaps, canMatch));
return fieldCaps;
}

static Map<String, List<String>> randomMappingHashToIndices() {
Map<String, List<String>> mappingHashToIndices = new HashMap<>();
int numGroups = between(0, 10);
for (int g = 0; g < numGroups; g++) {
String mappingHash = "mapping_hash_" + g;
String group = "group_" + g;
List<String> indices = IntStream.range(0, between(1, 10)).mapToObj(n -> group + "_index_" + n).toList();
mappingHashToIndices.put(mappingHash, indices);
}
Randomness.shuffle(inList);
final List<FieldCapabilitiesIndexResponse> serializedList;
try (BytesStreamOutput output = new BytesStreamOutput()) {
FieldCapabilitiesIndexResponse.writeList(output, inList);
try (
StreamInput in = new NamedWriteableAwareStreamInput(
output.bytes().streamInput(),
new NamedWriteableRegistry(Collections.emptyList())
)
) {
serializedList = FieldCapabilitiesIndexResponse.readList(in);
return mappingHashToIndices;
}

static List<FieldCapabilitiesIndexResponse> randomIndexResponsesWithMappingHash(Map<String, List<String>> mappingHashToIndices) {
final List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
for (Map.Entry<String, List<String>> e : mappingHashToIndices.entrySet()) {
Map<String, IndexFieldCapabilities> fieldCaps = randomFieldCaps();
String mappingHash = e.getKey();
for (String index : e.getValue()) {
responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, fieldCaps, true));
}
}
assertThat(
serializedList.stream().sorted(Comparator.comparing(FieldCapabilitiesIndexResponse::getIndexName)).toList(),
equalTo(inList.stream().sorted(Comparator.comparing(FieldCapabilitiesIndexResponse::getIndexName)).toList())
);
Map<String, List<FieldCapabilitiesIndexResponse>> groupedResponses = serializedList.stream()
.filter(r -> r.canMatch() && r.getIndexMappingHash() != null)
.collect(Collectors.groupingBy(FieldCapabilitiesIndexResponse::getIndexMappingHash));
assertThat(groupedResponses.keySet(), equalTo(mappingHashToIndices.keySet()));
for (Map.Entry<String, List<FieldCapabilitiesIndexResponse>> e : groupedResponses.entrySet()) {
List<String> indices = mappingHashToIndices.get(e.getKey());
List<FieldCapabilitiesIndexResponse> rs = e.getValue();
assertThat(rs.stream().map(FieldCapabilitiesIndexResponse::getIndexName).sorted().toList(), equalTo(indices));
for (FieldCapabilitiesIndexResponse r : rs) {
assertTrue(r.canMatch());
assertSame(r.get(), rs.get(0).get());
}
return responses;
}

static List<FieldCapabilitiesIndexResponse> randomIndexResponsesWithoutMappingHash() {
final List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
int numIndices = between(0, 10);
for (int i = 0; i < numIndices; i++) {
String index = "index_without_mapping_hash_" + i;
responses.add(new FieldCapabilitiesIndexResponse(index, null, randomFieldCaps(), randomBoolean()));
}
return responses;
}
}
167 changes: 163 additions & 4 deletions .../src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,37 @@

package org.elasticsearch.action.fieldcaps;

import org.elasticsearch.Version;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.test.VersionUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomFieldCaps;
import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomIndexResponsesWithMappingHash;
import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomIndexResponsesWithoutMappingHash;
import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponseTests.randomMappingHashToIndices;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.nullValue;

public class FieldCapabilitiesNodeResponseTests extends AbstractWireSerializingTestCase<FieldCapabilitiesNodeResponse> {

Expand All @@ -25,7 +47,7 @@ protected FieldCapabilitiesNodeResponse createTestInstance() {
List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
int numResponse = randomIntBetween(0, 10);
for (int i = 0; i < numResponse; i++) {
responses.add(FieldCapabilitiesResponseTests.createRandomIndexResponse());
responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, randomFieldCaps(), randomBoolean()));
}
int numUnmatched = randomIntBetween(0, 3);
Set<ShardId> shardIds = new HashSet<>();
Expand All @@ -43,18 +65,155 @@ protected Writeable.Reader<FieldCapabilitiesNodeResponse> instanceReader() {
@Override
protected FieldCapabilitiesNodeResponse mutateInstance(FieldCapabilitiesNodeResponse response) {
List<FieldCapabilitiesIndexResponse> newResponses = new ArrayList<>(response.getIndexResponses());
int mutation = response.getIndexResponses().isEmpty() ? 0 : randomIntBetween(0, 2);
int mutation = response.getIndexResponses().isEmpty() ? 0 : randomIntBetween(0, 3);
switch (mutation) {
case 0 -> newResponses.add(FieldCapabilitiesResponseTests.createRandomIndexResponse());
case 0 -> newResponses.add(new FieldCapabilitiesIndexResponse("extra_index", null, randomFieldCaps(), randomBoolean()));
case 1 -> {
int toRemove = randomInt(newResponses.size() - 1);
newResponses.remove(toRemove);
}
case 2 -> {
int toReplace = randomInt(newResponses.size() - 1);
newResponses.set(toReplace, FieldCapabilitiesResponseTests.createRandomIndexResponse());
newResponses.set(toReplace, new FieldCapabilitiesIndexResponse("new_index", null, randomFieldCaps(), randomBoolean()));
}
case 3 -> {
int toReplace = randomInt(newResponses.size() - 1);
FieldCapabilitiesIndexResponse resp = newResponses.get(toReplace);
newResponses.set(
toReplace,
new FieldCapabilitiesIndexResponse(resp.getIndexName(), UUIDs.randomBase64UUID(), resp.get(), true)
);
}
}
return new FieldCapabilitiesNodeResponse(newResponses, Collections.emptyMap(), response.getUnmatchedShardIds());
}

public void testSerializeNodeResponseBetweenNewNodes() throws Exception {
Map<String, List<String>> mappingHashToIndices = randomMappingHashToIndices();
List<FieldCapabilitiesIndexResponse> indexResponses = CollectionUtils.concatLists(
randomIndexResponsesWithMappingHash(mappingHashToIndices),
randomIndexResponsesWithoutMappingHash()
);
Randomness.shuffle(indexResponses);
FieldCapabilitiesNodeResponse inNode = randomNodeResponse(indexResponses);
final Version version = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.CURRENT);
final FieldCapabilitiesNodeResponse outNode = copyInstance(inNode, version);
assertThat(outNode.getFailures().keySet(), equalTo(inNode.getFailures().keySet()));
assertThat(outNode.getUnmatchedShardIds(), equalTo(inNode.getUnmatchedShardIds()));
final List<FieldCapabilitiesIndexResponse> inList = inNode.getIndexResponses();
final List<FieldCapabilitiesIndexResponse> outList = outNode.getIndexResponses();
assertThat(outList, hasSize(inList.size()));
assertThat(
outList.stream().sorted(Comparator.comparing(FieldCapabilitiesIndexResponse::getIndexName)).toList(),
equalTo(inList.stream().sorted(Comparator.comparing(FieldCapabilitiesIndexResponse::getIndexName)).toList())
);
Map<String, List<FieldCapabilitiesIndexResponse>> groupedResponses = outList.stream()
.filter(r -> r.canMatch() && r.getIndexMappingHash() != null)
.collect(Collectors.groupingBy(FieldCapabilitiesIndexResponse::getIndexMappingHash));
assertThat(groupedResponses.keySet(), equalTo(mappingHashToIndices.keySet()));
// Asserts responses of indices with the same mapping hash must be shared.
for (Map.Entry<String, List<FieldCapabilitiesIndexResponse>> e : groupedResponses.entrySet()) {
List<String> indices = mappingHashToIndices.get(e.getKey());
List<FieldCapabilitiesIndexResponse> rs = e.getValue();
assertThat(rs.stream().map(FieldCapabilitiesIndexResponse::getIndexName).sorted().toList(), equalTo(indices));
for (FieldCapabilitiesIndexResponse r : rs) {
assertTrue(r.canMatch());
assertSame(r.get(), rs.get(0).get());
}
}
}

public void testSerializeNodeResponseBetweenOldNodes() throws IOException {
final Version minCompactVersion = Version.CURRENT.minimumCompatibilityVersion();
assertTrue("Remove this test once minCompactVersion >= 8.2.0", minCompactVersion.before(Version.V_8_2_0));
List<FieldCapabilitiesIndexResponse> indexResponses = CollectionUtils.concatLists(
randomIndexResponsesWithMappingHash(randomMappingHashToIndices()),
randomIndexResponsesWithoutMappingHash()
);
Randomness.shuffle(indexResponses);
FieldCapabilitiesNodeResponse inResponse = randomNodeResponse(indexResponses);
Version version = VersionUtils.randomVersionBetween(random(), minCompactVersion, VersionUtils.getPreviousVersion(Version.V_8_2_0));
final FieldCapabilitiesNodeResponse outResponse = copyInstance(inResponse, version);
assertThat(outResponse.getFailures().keySet(), equalTo(inResponse.getFailures().keySet()));
assertThat(outResponse.getUnmatchedShardIds(), equalTo(inResponse.getUnmatchedShardIds()));
final List<FieldCapabilitiesIndexResponse> inList = inResponse.getIndexResponses();
final List<FieldCapabilitiesIndexResponse> outList = outResponse.getIndexResponses();
assertThat(outList, hasSize(inList.size()));
for (int i = 0; i < inList.size(); i++) {
assertThat("Responses between old nodes don't have mapping hash", outList.get(i).getIndexMappingHash(), nullValue());
assertThat(outList.get(i).getIndexName(), equalTo(inList.get(i).getIndexName()));
assertThat(outList.get(i).canMatch(), equalTo(inList.get(i).canMatch()));
Map<String, IndexFieldCapabilities> outCap = outList.get(i).get();
Map<String, IndexFieldCapabilities> inCap = inList.get(i).get();
if (version.onOrAfter(Version.V_8_0_0)) {
assertThat(outCap, equalTo(inCap));
} else {
// Exclude metric types which was introduced in 8.0
assertThat(outCap.keySet(), equalTo(inCap.keySet()));
for (String field : outCap.keySet()) {
assertThat(outCap.get(field).getName(), equalTo(inCap.get(field).getName()));
assertThat(outCap.get(field).getType(), equalTo(inCap.get(field).getType()));
assertThat(outCap.get(field).isSearchable(), equalTo(inCap.get(field).isSearchable()));
assertThat(outCap.get(field).isAggregatable(), equalTo(inCap.get(field).isAggregatable()));
assertThat(outCap.get(field).meta(), equalTo(inCap.get(field).meta()));
}
}
}
}

public void testReadNodeResponseFromPre82() throws Exception {
final Version minCompactVersion = Version.CURRENT.minimumCompatibilityVersion();
assertTrue("Remove this test once minCompactVersion >= 8.2.0", minCompactVersion.before(Version.V_8_2_0));
String base64 = "AwhpbmRleF8wMQIKYmx1ZV9maWVsZApibHVlX2ZpZWxkBGxvbmcAAQEAAAAJcmVkX2ZpZWxkCXJlZF9maWVsZAR0ZXh0AAEAAAAAAQhpbm"
+ "RleF8wMgAACGluZGV4XzAzAgdfc2VxX25vB19zZXFfbm8EbG9uZwEBAQAAAAx5ZWxsb3dfZmllbGQMeWVsbG93X2ZpZWxkB2tleXdvcmQAAQEAAAABAAEI"
+ "aW5kZXhfMTAGdXVpZF9hAQ==";
StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(base64));
in.setVersion(Version.V_8_1_0);
FieldCapabilitiesNodeResponse nodeResp = new FieldCapabilitiesNodeResponse(in);
assertThat(nodeResp.getUnmatchedShardIds(), equalTo(Set.of(new ShardId("index_10", "uuid_a", 1))));
assertThat(nodeResp.getFailures(), anEmptyMap());
assertThat(
nodeResp.getIndexResponses(),
contains(
new FieldCapabilitiesIndexResponse(
"index_01",
null,
Map.of(
"red_field",
new IndexFieldCapabilities("red_field", "text", false, true, false, false, null, Map.of()),
"blue_field",
new IndexFieldCapabilities("blue_field", "long", false, true, true, false, null, Map.of())
),
true
),
new FieldCapabilitiesIndexResponse("index_02", null, Map.of(), false),
new FieldCapabilitiesIndexResponse(
"index_03",
null,
Map.of(
"yellow_field",
new IndexFieldCapabilities("yellow_field", "keyword", false, true, true, false, null, Map.of()),
"_seq_no",
new IndexFieldCapabilities("_seq_no", "long", true, true, true, false, null, Map.of())
),
true
)
)
);
}

private static FieldCapabilitiesNodeResponse randomNodeResponse(List<FieldCapabilitiesIndexResponse> indexResponses) {
int numUnmatched = randomIntBetween(0, 3);
final Set<ShardId> unmatchedShardIds = new HashSet<>();
for (int i = 0; i < numUnmatched; i++) {
unmatchedShardIds.add(new ShardId(randomAlphaOfLength(10), randomAlphaOfLength(10), between(0, 10)));
}
final int failedShards = randomIntBetween(0, 3);
final Map<ShardId, Exception> failures = new HashMap<>();
for (int i = 0; i < failedShards; i++) {
ShardId shardId = new ShardId(randomAlphaOfLength(10), randomAlphaOfLength(10), between(0, 10));
failures.put(shardId, new IllegalStateException(randomAlphaOfLength(10)));
}
return new FieldCapabilitiesNodeResponse(indexResponses, failures, unmatchedShardIds);
}
}
Loading