Merge branch 'master' into primary-context

* master:
  Upgrade icu4j for the ICU analysis plugin to 59.1 (#25243)
  move assertBusy to use CheckException (#25246)
  Use SPI in High Level Rest Client to load XContent parsers (#25098)
  [TEST] test that low level REST client leaves path untouched (#25193)
  Speed up PK lookups at index time. (#19856)
  [Docs] Fix documentation for percentiles bucket aggregation (#25229)
  Upgrade to lucene-7.0.0-snapshot-92b1783. (#25222)
  Build: Add master flag for disabling bwc tests (#25230)
  Scripting: Rename SearchScript.needsScores to needs_score (#25235)
  Support script context stateful factory in Painless. (#25233)
  FastVectorHighlighter should not cache the field query globally (#25197)
  Remove QUERY_AND_FETCH BWC for pre-5.3.0 nodes (#25223)
  Add more missing AggregationBuilder getters (#25198)
  Extract the snapshot/restore full cluster restart tests from the translog full cluster restart tests (#25204)

Author: jasontedor

build.gradle

@@ -152,10 +152,28 @@ task verifyVersions {
   }
 }
 
+/*
+ * When adding backcompat behavior that spans major versions, temporarily
+ * disabling the backcompat tests is necessary. This flag controls
+ * the enabled state of every bwc task. It should be set back to true
+ * after the backport of the backcompat code is complete.
+ */
+allprojects {
+  ext.bwc_tests_enabled = true
+}
+
+task verifyBwcTestsEnabled {
+  doLast {
+    if (project.bwc_tests_enabled == false) {
+      throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
+    }
+  }
+}
+
 task branchConsistency {
   description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
   group 'Verification'
-  dependsOn verifyVersions
+  dependsOn verifyVersions, verifyBwcTestsEnabled
 }
 
 subprojects {

buildSrc/version.properties

@@ -1,6 +1,6 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 6.0.0-alpha3
-lucene            = 7.0.0-snapshot-a0aef2f
+lucene            = 7.0.0-snapshot-92b1783
 
 # optional dependencies
 spatial4j         = 0.6

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -49,8 +49,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
-import org.elasticsearch.join.aggregations.ParsedChildren;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -92,8 +91,6 @@
 import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.matrix.stats.ParsedMatrixStats;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.avg.ParsedAvg;
 import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
@@ -142,11 +139,13 @@
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -180,8 +179,9 @@ public RestHighLevelClient(RestClient restClient) {
      */
     protected RestHighLevelClient(RestClient restClient, List<NamedXContentRegistry.Entry> namedXContentEntries) {
         this.client = Objects.requireNonNull(restClient);
-        this.registry = new NamedXContentRegistry(Stream.of(getDefaultNamedXContents().stream(), namedXContentEntries.stream())
-                .flatMap(Function.identity()).collect(toList()));
+        this.registry = new NamedXContentRegistry(
+                Stream.of(getDefaultNamedXContents().stream(), getProvidedNamedXContents().stream(), namedXContentEntries.stream())
+                    .flatMap(Function.identity()).collect(toList()));
     }
 
     /**
@@ -566,8 +566,6 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
         map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
         map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
-        map.put(ChildrenAggregationBuilder.NAME, (p, c) -> ParsedChildren.fromXContent(p, (String) c));
-        map.put(MatrixStatsAggregationBuilder.NAME, (p, c) -> ParsedMatrixStats.fromXContent(p, (String) c));
         List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
                 .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
                 .collect(Collectors.toList());
@@ -579,4 +577,15 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
                 (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
         return entries;
     }
+
+    /**
+     * Loads and returns the {@link NamedXContentRegistry.Entry} parsers provided by plugins.
+     */
+    static List<NamedXContentRegistry.Entry> getProvidedNamedXContents() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+        for (NamedXContentProvider service : ServiceLoader.load(NamedXContentProvider.class)) {
+            entries.addAll(service.getNamedXContentParsers());
+        }
+        return entries;
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -56,10 +56,12 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -69,6 +71,7 @@
 
 import java.io.IOException;
 import java.net.SocketTimeoutException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -613,9 +616,9 @@ public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorValidBody
         assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage());
     }
 
-    public void testNamedXContents() {
+    public void testDefaultNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getDefaultNamedXContents();
-        assertEquals(45, namedXContents.size());
+        assertEquals(43, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
             Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1);
@@ -624,10 +627,28 @@ public void testNamedXContents() {
             }
         }
         assertEquals(2, categories.size());
-        assertEquals(Integer.valueOf(42), categories.get(Aggregation.class));
+        assertEquals(Integer.valueOf(40), categories.get(Aggregation.class));
         assertEquals(Integer.valueOf(3), categories.get(Suggest.Suggestion.class));
     }
 
+    public void testProvidedNamedXContents() {
+        List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
+        assertEquals(2, namedXContents.size());
+        Map<Class<?>, Integer> categories = new HashMap<>();
+        List<String> names = new ArrayList<>();
+        for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
+            names.add(namedXContent.name.getPreferredName());
+            Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1);
+            if (counter != null) {
+                categories.put(namedXContent.categoryClass, counter + 1);
+            }
+        }
+        assertEquals(1, categories.size());
+        assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
+        assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
+        assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+    }
+
     private static class TrackingActionListener implements ActionListener<Integer> {
         private final AtomicInteger statusCode = new AtomicInteger(-1);
         private final AtomicReference<Exception> exception = new AtomicReference<>();

client/rest/src/main/java/org/elasticsearch/client/RestClient.java

@@ -553,7 +553,7 @@ private static HttpRequestBase addRequestBody(HttpRequestBase httpRequest, HttpE
         return httpRequest;
     }
 
-    private static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
+    static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
         Objects.requireNonNull(path, "path must not be null");
         try {
             String fullPath;

client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java

@@ -23,6 +23,9 @@
 import org.apache.http.HttpHost;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 
+import java.net.URI;
+import java.util.Collections;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -77,6 +80,22 @@ public void testPerformAsyncWithWrongEndpoint() throws Exception {
         }
     }
 
+    public void testBuildUriLeavesPathUntouched() {
+        {
+            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", Collections.<String, String>emptyMap());
+            assertEquals("/foo$bar/index/type/id", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", Collections.<String, String>emptyMap());
+            assertEquals("/foo$bar/ty/pe/i/d", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/index/type/id", Collections.singletonMap("foo$bar", "x/y/z"));
+            assertEquals("/index/type/id", uri.getPath());
+            assertEquals("foo$bar=x/y/z", uri.getQuery());
+        }
+    }
+
     private static RestClient createRestClient() {
         HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)};
         return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), new Header[]{}, hosts, null, null);

core/licenses/lucene-analyzers-common-7.0.0-snapshot-92b1783.jar.sha1

@@ -0,0 +1 @@
+5bf8d8b7d885e25c343c187d1849580e21ef3fce

core/licenses/lucene-analyzers-common-7.0.0-snapshot-a0aef2f.jar.sha1

@@ -0,0 +1 @@
+9696b87e27ea949fabc62606833ab63e6e5473b9