Skip to content

Add tests for fields emulation CCS error cases #78621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed

Add tests for fields emulation CCS error cases #78621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6a633ac
Adding test for remote index with _source disabled
Sep 30, 2021
011a189
spotless
Jan 11, 2022
e8549d3
iter
Jan 11, 2022
73132b4
spotless
Jan 11, 2022
890a98a
spotless
Jan 11, 2022
dbde50d
fix test for 6.3 bwc nodes
Jan 11, 2022
f2518bc
iter
Jan 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 67 additions & 9 deletions ...n-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,20 @@
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.client.indices.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.rest.action.document.RestIndexAction;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.elasticsearch.test.rest.AbstractCCSRestTestCase;
import org.elasticsearch.test.rest.yaml.ObjectPath;
import org.elasticsearch.xcontent.DeprecationHandler;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xcontent.json.JsonXContent;

import java.io.IOException;
Expand All @@ -72,13 +75,10 @@ public class CCSFieldsOptionEmulationIT extends AbstractCCSRestTestCase {
private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
private static final String CLUSTER_ALIAS = "remote_cluster";

static int indexDocs(RestHighLevelClient client, String index, int numDocs, boolean expectWarnings) throws IOException {
static int indexDocs(RestHighLevelClient client, String index, int numDocs) throws IOException {
for (int i = 0; i < numDocs; i++) {
Request indexDoc = new Request("PUT", index + "/type/" + i);
Request indexDoc = new Request("PUT", index + "/_doc/" + i);
indexDoc.setJsonEntity("{\"field\": \"f" + i + "\", \"array\": [1, 2, 3] , \"obj\": { \"innerObj\" : \"foo\" } }");
if (expectWarnings) {
indexDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
}
client.getLowLevelClient().performRequest(indexDoc);
}
client.indices().refresh(new RefreshRequest(index), RequestOptions.DEFAULT);
Expand Down Expand Up @@ -107,14 +107,29 @@ public void testFieldsOptionEmulation() throws Exception {
),
RequestOptions.DEFAULT
);
int localNumDocs = indexDocs(localClient, localIndex, between(10, 20), true);
int localNumDocs = indexDocs(localClient, localIndex, between(10, 20));

Builder remoteIndexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5));
remoteClient.indices().create(new CreateIndexRequest(remoteIndex).settings(remoteIndexSettings), RequestOptions.DEFAULT);
boolean expectRemoteIndexWarnings = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_0_0);
int remoteNumDocs = indexDocs(remoteClient, remoteIndex, between(10, 20), expectRemoteIndexWarnings);
int remoteNumDocs = indexDocs(remoteClient, remoteIndex, between(10, 20));
int expectedHitCount = localNumDocs + remoteNumDocs;

// create a second remote index with "_source : false" to check error behaviour in this case
String remoteNoSourceIndex = "remote_index_source_disabled";
String mappings = "{\"_source\":{\"enabled\":false}}";
if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
mappings = "{\"_doc\":" + mappings + "}";
}
remoteClient.indices()
.create(
new CreateIndexRequest(remoteNoSourceIndex).settings(remoteIndexSettings).mapping(mappings, XContentType.JSON),
RequestOptions.DEFAULT
);
GetMappingsResponse mapping = remoteClient.indices()
.getMapping(new GetMappingsRequest().indices(remoteNoSourceIndex), RequestOptions.DEFAULT);
assertEquals(mappings, mapping.mappings().get(remoteNoSourceIndex).source().string());
int remoteNoSourceNumDocs = indexDocs(remoteClient, remoteNoSourceIndex, between(10, 20));

List<Node> remoteNodes = getNodes(remoteClient.getLowLevelClient());
assertThat(remoteNodes, hasSize(2));
configureRemoteClusters(getNodes(remoteClient.getLowLevelClient()), CLUSTER_ALIAS, UPGRADE_FROM_VERSION, LOGGER);
Expand Down Expand Up @@ -159,6 +174,48 @@ public void testFieldsOptionEmulation() throws Exception {
assertEquals("foo", fields.get("obj.innerObj").getValue());
}
}

// check errors when index with disabled _source is included
request = new Request("POST", "/_search");
request.addParameter(
"index",
localIndex + "," + CLUSTER_ALIAS + ":" + remoteIndex + "," + CLUSTER_ALIAS + ":" + remoteNoSourceIndex
);
request.addParameter("ccs_minimize_roundtrips", minimizeRoundTrips);
request.addParameter("enable_fields_emulation", "true");
request.setJsonEntity(
"{\"_source\": false, \"fields\": [\"*\"] , \"size\": " + expectedHitCount + remoteNoSourceNumDocs + "}"
);
response = lowLevelClient.performRequest(request);
ObjectPath responseObject = ObjectPath.createFromResponse(response);
List<Map<String, Object>> failures = responseObject.evaluate("_shards.failures");
assertEquals(1, failures.size());
assertEquals(CLUSTER_ALIAS + ":" + remoteNoSourceIndex, responseObject.evaluate("_shards.failures.0.index"));
assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type"));
assertThat(
responseObject.evaluate("_shards.failures.0.reason.reason"),
containsString("_source is disabled in the mappings for index [remote_index_source_disabled]")
);
assertEquals(expectedHitCount, ((List<?>) responseObject.evaluate("hits.hits")).size());

// also check for only no-source remote index
request = new Request("POST", "/_search");
request.addParameter("index", localIndex + "," + CLUSTER_ALIAS + ":" + remoteNoSourceIndex);
request.addParameter("ccs_minimize_roundtrips", minimizeRoundTrips);
request.addParameter("enable_fields_emulation", "true");
request.setJsonEntity("{\"_source\": false, \"fields\": [\"*\"] , \"size\": " + localNumDocs + remoteNoSourceNumDocs + "}");
response = lowLevelClient.performRequest(request);
responseObject = ObjectPath.createFromResponse(response);
failures = responseObject.evaluate("_shards.failures");
assertEquals(1, failures.size());
assertEquals(CLUSTER_ALIAS + ":" + remoteNoSourceIndex, responseObject.evaluate("_shards.failures.0.index"));
assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type"));
assertThat(
responseObject.evaluate("_shards.failures.0.reason.reason"),
containsString("_source is disabled in the mappings for index [remote_index_source_disabled]")
);
// all local hits should still be there
assertEquals(localNumDocs, ((List<?>) responseObject.evaluate("hits.hits")).size());
}

// also check validation of request
Expand All @@ -177,6 +234,7 @@ public void testFieldsOptionEmulation() throws Exception {

localClient.indices().delete(new DeleteIndexRequest(localIndex), RequestOptions.DEFAULT);
remoteClient.indices().delete(new DeleteIndexRequest(remoteIndex), RequestOptions.DEFAULT);
remoteClient.indices().delete(new DeleteIndexRequest(remoteNoSourceIndex), RequestOptions.DEFAULT);
}
}
}
60 changes: 49 additions & 11 deletions qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/FieldsOptionEmulationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ObjectPath;
Expand All @@ -25,6 +26,8 @@
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.containsString;

/**
* In mixed cluster scenarios on 7.x we try to emulate the "fields" option introduced in 7.10
* by running a request with "source" enabled for the requested patterns on older nodes and convert
Expand All @@ -35,6 +38,7 @@ public class FieldsOptionEmulationIT extends ESRestTestCase {

private static String index = "test_field_newversion";
private static String index_old = "test_field_oldversion";
private static String index_old_no_source = "oldversion_no_source";
private static Nodes nodes;
private static List<Node> bwcNodes;
private static List<Node> newNodes;
Expand All @@ -50,21 +54,38 @@ public void prepareTestData() throws IOException {
oldNodeName = bwcNodes.get(0).getNodeName();
newNodeName = newNodes.get(0).getNodeName();
bwcNodeVersion = bwcNodes.get(0).getVersion();
createIndexOnNode(index, newNodeName);
createIndexOnNode(index_old, oldNodeName);
createIndexOnNode(index, newNodeName, null, false);
createIndexOnNode(index_old, oldNodeName, null, false);
String sourceDisabledMapping = "\"_source\":{\"enabled\": false}";
boolean includeTypeName = bwcNodeVersion.before(Version.V_7_0_0);
if (includeTypeName) {
sourceDisabledMapping = "\"_doc\": {" + sourceDisabledMapping + "}";
}
createIndexOnNode(index_old_no_source, oldNodeName, sourceDisabledMapping, includeTypeName);
refreshAllIndices();
}

private void createIndexOnNode(String indexName, String nodeName) throws IOException {
private void createIndexOnNode(String indexName, String nodeName, String mappings, boolean includeTypeName) throws IOException {
if (indexExists(indexName) == false) {
createIndex(
indexName,
Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", nodeName)
.build()
);
Request createIndexRequest = new Request("PUT", "/" + indexName);
String entity = "{\"settings\": "
+ Strings.toString(
Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", nodeName)
.build()
);
if (mappings != null) {
entity += ",\"mappings\" : {" + mappings + "}";
}
entity += "}";
createIndexRequest.setJsonEntity(entity);
createIndexRequest.setOptions(allowTypesRemovalWarnings());
if (includeTypeName) {
createIndexRequest.addParameter("include_type_name", "true");
}
client().performRequest(createIndexRequest);
for (int i = 0; i < 5; i++) {
Request request = new Request("PUT", indexName + "/_doc/" + i);
request.setJsonEntity(
Expand Down Expand Up @@ -202,4 +223,21 @@ public void testGettingObjects() throws Exception {
}
}
}

public void testFieldOptionNoSourceOnOldIndex() throws Exception {
Request matchAllRequestFiltered = new Request("POST", "test_field_*,oldversion_no_source/_search");
matchAllRequestFiltered.addParameter("enable_fields_emulation", "true");
matchAllRequestFiltered.setJsonEntity("{\"_source\":false,\"fields\":[\"*\"]}");
try (
RestClient client = buildClient(restClientSettings(), newNodes.stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))
) {
Response response = client.performRequest(matchAllRequestFiltered);
ObjectPath responseObject = ObjectPath.createFromResponse(response);
List<Map<String, Object>> failures = responseObject.evaluate("_shards.failures");
assertEquals(1, failures.size());
assertEquals(index_old_no_source, responseObject.evaluate("_shards.failures.0.index"));
assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type"));
assertThat(responseObject.evaluate("_shards.failures.0.reason.reason"), containsString("_source is disabled"));
}
}
}