Merge branch 'master' into ccr

* master:
  Add proxy support to RemoteClusterConnection (#33062)
  TEST: Skip assertSeqNos for closed shards (#33130)
  TEST: resync operation on replica should acquire shard permit (#33103)
  Switch remaining x-pack tests to new style Requests (#33108)
  Switch remaining tests to new style Requests (#33109)
  Switch remaining ml tests to new style Requests (#33107)
  Build: Line up IDE detection logic
  Security index expands to a single replica (#33131)
  HLRC: request/response homogeneity and JavaDoc improvements (#33133)
  Checkstyle!
  [Test] Fix sporadic failure in MembershipActionTests
  Revert "Do NOT allow termvectors on nested fields (#32728)"
  [Rollup] Move toAggCap() methods out of rollup config objects (#32583)
  Fix race condition in scheduler engine test

Author: dnhatn

client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java

@@ -17,9 +17,6 @@
  * under the License.
  */
 
-
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
@@ -35,7 +32,6 @@
 
 import java.util.Collections;
 
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -52,12 +48,9 @@ public void testGetStoredScript() throws Exception {
         final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
         // TODO: change to HighLevel PutStoredScriptRequest when it will be ready
         // so far - using low-level REST API
-        Response putResponse =
-            adminClient()
-                .performRequest("PUT", "/_scripts/calculate-score", emptyMap(),
-                    new StringEntity("{\"script\":" + script + "}",
-                        ContentType.APPLICATION_JSON));
-        assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
+        Request putRequest = new Request("PUT", "/_scripts/calculate-score");
+        putRequest.setJsonEntity("{\"script\":" + script + "}");
+        Response putResponse = adminClient().performRequest(putRequest);
         assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));
 
         GetStoredScriptRequest getRequest = new GetStoredScriptRequest("calculate-score");
@@ -78,12 +71,9 @@ public void testDeleteStoredScript() throws Exception {
         final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
         // TODO: change to HighLevel PutStoredScriptRequest when it will be ready
         // so far - using low-level REST API
-        Response putResponse =
-            adminClient()
-                .performRequest("PUT", "/_scripts/" + id, emptyMap(),
-                    new StringEntity("{\"script\":" + script + "}",
-                        ContentType.APPLICATION_JSON));
-        assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
+        Request putRequest = new Request("PUT", "/_scripts/" + id);
+        putRequest.setJsonEntity("{\"script\":" + script + "}");
+        Response putResponse = adminClient().performRequest(putRequest);
         assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));
 
         DeleteStoredScriptRequest deleteRequest = new DeleteStoredScriptRequest(id);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java

@@ -17,8 +17,6 @@
  * under the License.
  */
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
@@ -27,6 +25,7 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -43,7 +42,6 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -193,11 +191,9 @@ private void putStoredScript(String id, StoredScriptSource scriptSource) throws
         final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
         // TODO: change to HighLevel PutStoredScriptRequest when it will be ready
         // so far - using low-level REST API
-        Response putResponse =
-            adminClient()
-                .performRequest("PUT", "/_scripts/" + id, emptyMap(),
-                    new StringEntity("{\"script\":" + script + "}",
-                        ContentType.APPLICATION_JSON));
+        Request request = new Request("PUT", "/_scripts/" + id);
+        request.setJsonEntity("{\"script\":" + script + "}");
+        Response putResponse = adminClient().performRequest(request);
         assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
         assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));
     }

distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java

@@ -53,7 +53,7 @@ public void setupIndex() throws IOException {
 
     @After
     public void cleanupIndex() throws IOException {
-        client().performRequest("DELETE", indexName());
+        client().performRequest(new Request("DELETE", indexName()));
     }
 
     private String indexName() {

docs/reference/docs/termvectors.asciidoc

@@ -30,10 +30,6 @@ in similar way to the <<query-dsl-multi-match-query,multi match query>>
 [WARNING]
 Note that the usage of `/_termvector` is deprecated in 2.0, and replaced by `/_termvectors`.
 
-[WARNING]
-Term Vectors API doesn't work on nested fields. `/_termvectors` on a nested
-field and any sub-fields of a nested field returns empty results.
-
 [float]
 === Return values
 

rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_nested.yml

@@ -272,6 +272,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
                     TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
                     RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
+                    RemoteClusterAware.REMOTE_CLUSTERS_PROXY,
                     RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
                     RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER,
                     RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -1009,6 +1009,10 @@ public static Setting<String> simpleString(String key, Property... properties) {
         return new Setting<>(key, s -> "", Function.identity(), properties);
     }
 
+    public static Setting<String> simpleString(String key, Function<String, String> parser, Property... properties) {
+        return new Setting<>(key, s -> "", parser, properties);
+    }
+
     public static Setting<String> simpleString(String key, Setting<String> fallback, Property... properties) {
         return new Setting<>(key, fallback, Function.identity(), properties);
     }

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -161,7 +160,7 @@ private static void handleFieldWildcards(IndexShard indexShard, TermVectorsReque
         request.selectedFields(fieldNames.toArray(Strings.EMPTY_ARRAY));
     }
 
-    private static boolean isValidField(MappedFieldType fieldType, IndexShard indexShard) {
+    private static boolean isValidField(MappedFieldType fieldType) {
         // must be a string
         if (fieldType instanceof StringFieldType == false) {
             return false;
@@ -170,16 +169,6 @@ private static boolean isValidField(MappedFieldType fieldType, IndexShard indexS
         if (fieldType.indexOptions() == IndexOptions.NONE) {
             return false;
         }
-        // and must not be under nested field
-        int dotIndex = fieldType.name().indexOf('.');
-        while (dotIndex > -1) {
-            String parentField = fieldType.name().substring(0, dotIndex);
-            ObjectMapper mapper = indexShard.mapperService().getObjectMapper(parentField);
-            if (mapper != null && mapper.nested().isNested()) {
-                return false;
-            }
-            dotIndex = fieldType.name().indexOf('.', dotIndex + 1);
-        }
         return true;
     }
 
@@ -188,7 +177,7 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR
         Set<String> validFields = new HashSet<>();
         for (String field : selectedFields) {
             MappedFieldType fieldType = indexShard.mapperService().fullName(field);
-            if (isValidField(fieldType, indexShard) == false) {
+            if (!isValidField(fieldType)) {
                 continue;
             }
             // already retrieved, only if the analyzer hasn't been overridden at the field
@@ -295,7 +284,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
         Collection<DocumentField> documentFields = new HashSet<>();
         for (IndexableField field : doc.getFields()) {
             MappedFieldType fieldType = indexShard.mapperService().fullName(field.name());
-            if (isValidField(fieldType, indexShard) == false) {
+            if (!isValidField(fieldType)) {
                 continue;
             }
             if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {

server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java

@@ -18,10 +18,14 @@
  */
 package org.elasticsearch.transport;
 
+import java.util.EnumSet;
 import java.util.function.Supplier;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -66,6 +70,22 @@ public abstract class RemoteClusterAware extends AbstractComponent {
     public static final char REMOTE_CLUSTER_INDEX_SEPARATOR = ':';
     public static final String LOCAL_CLUSTER_GROUP_KEY = "";
 
+    /**
+     * A proxy address for the remote cluster.
+     * NOTE: this settings is undocumented until we have at last one transport that supports passing
+     * on the hostname via a mechanism like SNI.
+     */
+    public static final Setting.AffixSetting<String> REMOTE_CLUSTERS_PROXY = Setting.affixKeySetting(
+        "search.remote.",
+        "proxy",
+        key -> Setting.simpleString(key, s -> {
+            if (Strings.hasLength(s)) {
+                parsePort(s);
+            }
+            return s;
+        }, Setting.Property.NodeScope, Setting.Property.Dynamic), REMOTE_CLUSTERS_SEEDS);
+
+
     protected final ClusterNameExpressionResolver clusterNameResolver;
 
     /**
@@ -77,25 +97,42 @@ protected RemoteClusterAware(Settings settings) {
         this.clusterNameResolver = new ClusterNameExpressionResolver(settings);
     }
 
-    protected static Map<String, List<Supplier<DiscoveryNode>>> buildRemoteClustersSeeds(Settings settings) {
+    /**
+     * Builds the dynamic per-cluster config from the given settings. This is a map keyed by the cluster alias that points to a tuple
+     * (ProxyAddresss, [SeedNodeSuppliers]). If a cluster is configured with a proxy address all seed nodes will point to
+     * {@link TransportAddress#META_ADDRESS} and their configured address will be used as the hostname for the generated discovery node.
+     */
+    protected static Map<String, Tuple<String, List<Supplier<DiscoveryNode>>>> buildRemoteClustersDynamicConfig(Settings settings) {
         Stream<Setting<List<String>>> allConcreteSettings = REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings);
         return allConcreteSettings.collect(
             Collectors.toMap(REMOTE_CLUSTERS_SEEDS::getNamespace, concreteSetting -> {
                 String clusterName = REMOTE_CLUSTERS_SEEDS.getNamespace(concreteSetting);
                 List<String> addresses = concreteSetting.get(settings);
+                final boolean proxyMode = REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).exists(settings);
                 List<Supplier<DiscoveryNode>> nodes = new ArrayList<>(addresses.size());
                 for (String address : addresses) {
-                    nodes.add(() -> {
-                        TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address));
-                        return new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
-                            transportAddress,
-                            Version.CURRENT.minimumCompatibilityVersion());
-                    });
+                    nodes.add(() -> buildSeedNode(clusterName, address, proxyMode));
                 }
-                return nodes;
+                return new Tuple<>(REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).get(settings), nodes);
             }));
     }
 
+    static DiscoveryNode buildSeedNode(String clusterName, String address, boolean proxyMode) {
+        if (proxyMode) {
+            TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 0);
+            String hostName = address.substring(0, indexOfPortSeparator(address));
+            return new DiscoveryNode("", clusterName + "#" + address, UUIDs.randomBase64UUID(), hostName, address,
+                transportAddress, Collections
+                .emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class),
+                Version.CURRENT.minimumCompatibilityVersion());
+        } else {
+            TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address));
+            return new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
+                transportAddress,
+                Version.CURRENT.minimumCompatibilityVersion());
+        }
+    }
+
     /**
      * Groups indices per cluster by splitting remote cluster-alias, index-name pairs on {@link #REMOTE_CLUSTER_INDEX_SEPARATOR}. All
      * indices per cluster are collected as a list in the returned map keyed by the cluster alias. Local indices are grouped under
@@ -138,20 +175,24 @@ public Map<String, List<String>> groupClusterIndices(String[] requestIndices, Pr
 
     protected abstract Set<String> getRemoteClusterNames();
 
+
     /**
      * Subclasses must implement this to receive information about updated cluster aliases. If the given address list is
      * empty the cluster alias is unregistered and should be removed.
      */
-    protected abstract void updateRemoteCluster(String clusterAlias, List<String> addresses);
+    protected abstract void updateRemoteCluster(String clusterAlias, List<String> addresses, String proxy);
 
     /**
      * Registers this instance to listen to updates on the cluster settings.
      */
     public void listenForUpdates(ClusterSettings clusterSettings) {
-        clusterSettings.addAffixUpdateConsumer(RemoteClusterAware.REMOTE_CLUSTERS_SEEDS, this::updateRemoteCluster,
+        clusterSettings.addAffixUpdateConsumer(RemoteClusterAware.REMOTE_CLUSTERS_PROXY,
+            RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
+            (key, value) -> updateRemoteCluster(key, value.v2(), value.v1()),
             (namespace, value) -> {});
     }
 
+
     protected static InetSocketAddress parseSeedAddress(String remoteHost) {
         String host = remoteHost.substring(0, indexOfPortSeparator(remoteHost));
         InetAddress hostAddress;