diff --git a/qa/ccs-old-version-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java b/qa/ccs-old-version-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java index e5546460a13ed..4db25c4434240 100644 --- a/qa/ccs-old-version-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java +++ b/qa/ccs-old-version-remote-cluster/src/test/java/org/elasticsearch/upgrades/CCSFieldsOptionEmulationIT.java @@ -41,17 +41,20 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; -import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.test.rest.AbstractCCSRestTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; @@ -72,13 +75,10 @@ public class CCSFieldsOptionEmulationIT extends AbstractCCSRestTestCase { private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); private static final String CLUSTER_ALIAS = "remote_cluster"; - static int indexDocs(RestHighLevelClient client, String index, int numDocs, boolean expectWarnings) throws IOException { + static int indexDocs(RestHighLevelClient client, String index, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { - Request indexDoc = new Request("PUT", index + "/type/" + i); + Request indexDoc = new Request("PUT", index + "/_doc/" + i); indexDoc.setJsonEntity("{\"field\": \"f" + i + "\", \"array\": [1, 2, 3] , \"obj\": { \"innerObj\" : \"foo\" } }"); - if (expectWarnings) { - indexDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); - } client.getLowLevelClient().performRequest(indexDoc); } client.indices().refresh(new RefreshRequest(index), RequestOptions.DEFAULT); @@ -107,14 +107,29 @@ public void testFieldsOptionEmulation() throws Exception { ), RequestOptions.DEFAULT ); - int localNumDocs = indexDocs(localClient, localIndex, between(10, 20), true); + int localNumDocs = indexDocs(localClient, localIndex, between(10, 20)); Builder remoteIndexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)); remoteClient.indices().create(new CreateIndexRequest(remoteIndex).settings(remoteIndexSettings), RequestOptions.DEFAULT); - boolean expectRemoteIndexWarnings = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_0_0); - int remoteNumDocs = indexDocs(remoteClient, remoteIndex, between(10, 20), expectRemoteIndexWarnings); + int remoteNumDocs = indexDocs(remoteClient, remoteIndex, between(10, 20)); int expectedHitCount = localNumDocs + remoteNumDocs; + // create a second remote index with "_source : false" to check error behaviour in this case + String remoteNoSourceIndex = "remote_index_source_disabled"; + String mappings = "{\"_source\":{\"enabled\":false}}"; + if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) { + mappings = "{\"_doc\":" + mappings + "}"; + } + remoteClient.indices() + .create( + new CreateIndexRequest(remoteNoSourceIndex).settings(remoteIndexSettings).mapping(mappings, XContentType.JSON), + RequestOptions.DEFAULT + ); + GetMappingsResponse mapping = remoteClient.indices() + .getMapping(new GetMappingsRequest().indices(remoteNoSourceIndex), RequestOptions.DEFAULT); + assertEquals(mappings, mapping.mappings().get(remoteNoSourceIndex).source().string()); + int remoteNoSourceNumDocs = indexDocs(remoteClient, remoteNoSourceIndex, between(10, 20)); + List remoteNodes = getNodes(remoteClient.getLowLevelClient()); assertThat(remoteNodes, hasSize(2)); configureRemoteClusters(getNodes(remoteClient.getLowLevelClient()), CLUSTER_ALIAS, UPGRADE_FROM_VERSION, LOGGER); @@ -159,6 +174,48 @@ public void testFieldsOptionEmulation() throws Exception { assertEquals("foo", fields.get("obj.innerObj").getValue()); } } + + // check errors when index with disabled _source is included + request = new Request("POST", "/_search"); + request.addParameter( + "index", + localIndex + "," + CLUSTER_ALIAS + ":" + remoteIndex + "," + CLUSTER_ALIAS + ":" + remoteNoSourceIndex + ); + request.addParameter("ccs_minimize_roundtrips", minimizeRoundTrips); + request.addParameter("enable_fields_emulation", "true"); + request.setJsonEntity( + "{\"_source\": false, \"fields\": [\"*\"] , \"size\": " + expectedHitCount + remoteNoSourceNumDocs + "}" + ); + response = lowLevelClient.performRequest(request); + ObjectPath responseObject = ObjectPath.createFromResponse(response); + List> failures = responseObject.evaluate("_shards.failures"); + assertEquals(1, failures.size()); + assertEquals(CLUSTER_ALIAS + ":" + remoteNoSourceIndex, responseObject.evaluate("_shards.failures.0.index")); + assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type")); + assertThat( + responseObject.evaluate("_shards.failures.0.reason.reason"), + containsString("_source is disabled in the mappings for index [remote_index_source_disabled]") + ); + assertEquals(expectedHitCount, ((List) responseObject.evaluate("hits.hits")).size()); + + // also check for only no-source remote index + request = new Request("POST", "/_search"); + request.addParameter("index", localIndex + "," + CLUSTER_ALIAS + ":" + remoteNoSourceIndex); + request.addParameter("ccs_minimize_roundtrips", minimizeRoundTrips); + request.addParameter("enable_fields_emulation", "true"); + request.setJsonEntity("{\"_source\": false, \"fields\": [\"*\"] , \"size\": " + localNumDocs + remoteNoSourceNumDocs + "}"); + response = lowLevelClient.performRequest(request); + responseObject = ObjectPath.createFromResponse(response); + failures = responseObject.evaluate("_shards.failures"); + assertEquals(1, failures.size()); + assertEquals(CLUSTER_ALIAS + ":" + remoteNoSourceIndex, responseObject.evaluate("_shards.failures.0.index")); + assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type")); + assertThat( + responseObject.evaluate("_shards.failures.0.reason.reason"), + containsString("_source is disabled in the mappings for index [remote_index_source_disabled]") + ); + // all local hits should still be there + assertEquals(localNumDocs, ((List) responseObject.evaluate("hits.hits")).size()); } // also check validation of request @@ -177,6 +234,7 @@ public void testFieldsOptionEmulation() throws Exception { localClient.indices().delete(new DeleteIndexRequest(localIndex), RequestOptions.DEFAULT); remoteClient.indices().delete(new DeleteIndexRequest(remoteIndex), RequestOptions.DEFAULT); + remoteClient.indices().delete(new DeleteIndexRequest(remoteNoSourceIndex), RequestOptions.DEFAULT); } } } diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/FieldsOptionEmulationIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/FieldsOptionEmulationIT.java index 11f8cb71bcd08..37693e3f3602a 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/FieldsOptionEmulationIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/FieldsOptionEmulationIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; @@ -25,6 +26,8 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; + /** * In mixed cluster scenarios on 7.x we try to emulate the "fields" option introduced in 7.10 * by running a request with "source" enabled for the requested patterns on older nodes and convert @@ -35,6 +38,7 @@ public class FieldsOptionEmulationIT extends ESRestTestCase { private static String index = "test_field_newversion"; private static String index_old = "test_field_oldversion"; + private static String index_old_no_source = "oldversion_no_source"; private static Nodes nodes; private static List bwcNodes; private static List newNodes; @@ -50,21 +54,38 @@ public void prepareTestData() throws IOException { oldNodeName = bwcNodes.get(0).getNodeName(); newNodeName = newNodes.get(0).getNodeName(); bwcNodeVersion = bwcNodes.get(0).getVersion(); - createIndexOnNode(index, newNodeName); - createIndexOnNode(index_old, oldNodeName); + createIndexOnNode(index, newNodeName, null, false); + createIndexOnNode(index_old, oldNodeName, null, false); + String sourceDisabledMapping = "\"_source\":{\"enabled\": false}"; + boolean includeTypeName = bwcNodeVersion.before(Version.V_7_0_0); + if (includeTypeName) { + sourceDisabledMapping = "\"_doc\": {" + sourceDisabledMapping + "}"; + } + createIndexOnNode(index_old_no_source, oldNodeName, sourceDisabledMapping, includeTypeName); refreshAllIndices(); } - private void createIndexOnNode(String indexName, String nodeName) throws IOException { + private void createIndexOnNode(String indexName, String nodeName, String mappings, boolean includeTypeName) throws IOException { if (indexExists(indexName) == false) { - createIndex( - indexName, - Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", nodeName) - .build() - ); + Request createIndexRequest = new Request("PUT", "/" + indexName); + String entity = "{\"settings\": " + + Strings.toString( + Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", nodeName) + .build() + ); + if (mappings != null) { + entity += ",\"mappings\" : {" + mappings + "}"; + } + entity += "}"; + createIndexRequest.setJsonEntity(entity); + createIndexRequest.setOptions(allowTypesRemovalWarnings()); + if (includeTypeName) { + createIndexRequest.addParameter("include_type_name", "true"); + } + client().performRequest(createIndexRequest); for (int i = 0; i < 5; i++) { Request request = new Request("PUT", indexName + "/_doc/" + i); request.setJsonEntity( @@ -202,4 +223,21 @@ public void testGettingObjects() throws Exception { } } } + + public void testFieldOptionNoSourceOnOldIndex() throws Exception { + Request matchAllRequestFiltered = new Request("POST", "test_field_*,oldversion_no_source/_search"); + matchAllRequestFiltered.addParameter("enable_fields_emulation", "true"); + matchAllRequestFiltered.setJsonEntity("{\"_source\":false,\"fields\":[\"*\"]}"); + try ( + RestClient client = buildClient(restClientSettings(), newNodes.stream().map(Node::getPublishAddress).toArray(HttpHost[]::new)) + ) { + Response response = client.performRequest(matchAllRequestFiltered); + ObjectPath responseObject = ObjectPath.createFromResponse(response); + List> failures = responseObject.evaluate("_shards.failures"); + assertEquals(1, failures.size()); + assertEquals(index_old_no_source, responseObject.evaluate("_shards.failures.0.index")); + assertEquals("illegal_argument_exception", responseObject.evaluate("_shards.failures.0.reason.type")); + assertThat(responseObject.evaluate("_shards.failures.0.reason.reason"), containsString("_source is disabled")); + } + } }