Merge branch 'upstream/master' into rollover-add-max_shard_docs

* upstream/master:
  [DOCS] Switch xrefs to external links (elastic#83590)
  [DOCS] 'features' flag added in elastic#83083 (elastic#83452)
  Rename ChangePolicyforIndexIT to ChangePolicyForIndexIT (elastic#83569)
  Fixing random_sampler tests (elastic#83549)
  Upgrade Checkstyle to 9.3 (elastic#83314)
  Make improvements to the release notes generator (elastic#83525)
  Cleanup DataTierAllocationDecider (elastic#83572)
  Upgrade jANSI dependency to 2.4.0 (elastic#83566)
  Speed up Name Collision Check in Metadata.Builder (elastic#83340)
  SQL: Add range checks to interval multiplication operation (elastic#83478)
  Remove DiscoveryNodes#getAllNodes (elastic#83538)
  Make RoutingNodes behave like a collection (elastic#83540)
  Remove Unused CS Listener from SecurityServerTransportInterceptor (elastic#83556)

Author: weizijun

.idea/checkstyle-idea.xml

@@ -156,7 +156,8 @@ public ClusterState measureAllocation() {
         while (clusterState.getRoutingNodes().hasUnassignedShards()) {
             clusterState = strategy.applyStartedShards(
                 clusterState,
-                StreamSupport.stream(clusterState.getRoutingNodes().spliterator(), false)
+                clusterState.getRoutingNodes()
+                    .stream()
                     .flatMap(shardRoutings -> StreamSupport.stream(shardRoutings.spliterator(), false))
                     .filter(ShardRouting::initializing)
                     .collect(Collectors.toList())

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -65,7 +65,7 @@ dependencies {
     api 'org.apache.maven:maven-model:3.6.2'
     api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
     api 'org.apache.rat:apache-rat:0.11'
-    compileOnly "com.puppycrawl.tools:checkstyle:8.45.1"
+    compileOnly "com.puppycrawl.tools:checkstyle:9.3"
     api('com.diffplug.spotless:spotless-plugin-gradle:6.0.0') {
       exclude module: "groovy-xml"
     }

build-conventions/build.gradle

@@ -2,17 +2,17 @@
  * @notice
  * checkstyle: Checks Java source code for adherence to a set of rules.
  * Copyright (C) 2001-2021 the original author or authors.
-
+ *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
-
+ *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
-
+ *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/HiddenFieldCheck.java

@@ -121,6 +121,8 @@ public int[] getRequiredTokens() {
         return CommonUtil.EMPTY_INT_ARRAY;
     }
 
+    // suppress deprecation until https://github.com/checkstyle/checkstyle/issues/11166
+    @SuppressWarnings("deprecation")
     @Override
     public void visitToken(DetailAST ast) {
         if (shouldCheck(ast)) {
@@ -140,24 +142,18 @@ public void visitToken(DetailAST ast) {
      * @return whether we should check a given node.
      */
     private boolean shouldCheck(final DetailAST ast) {
-        final Scope customScope;
-
-        if (ScopeUtil.isInInterfaceOrAnnotationBlock(ast)) {
-            customScope = Scope.PUBLIC;
-        } else {
-            final DetailAST mods = ast.findFirstToken(TokenTypes.MODIFIERS);
-            customScope = ScopeUtil.getScopeFromMods(mods);
-        }
+        final Scope customScope = ScopeUtil.getScope(ast);
         final Scope surroundingScope = ScopeUtil.getSurroundingScope(ast);
 
         final String outerTypeName = ast.findFirstToken(TokenTypes.IDENT).getText();
 
         return customScope.isIn(scope)
             && (surroundingScope == null || surroundingScope.isIn(scope))
             && (excludeScope == null
-                || customScope.isIn(excludeScope) == false
-                || surroundingScope != null && surroundingScope.isIn(excludeScope) == false)
-            && AnnotationUtil.containsAnnotation(ast, skipAnnotations) == false
+                || !customScope.isIn(excludeScope)
+                || surroundingScope != null
+                && !surroundingScope.isIn(excludeScope))
+            && !AnnotationUtil.containsAnnotation(ast, skipAnnotations)
             && ignorePattern.matcher(outerTypeName).find() == false;
     }
 

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java

@@ -42,7 +42,7 @@
 import java.util.stream.StreamSupport;
 
 /**
- * Incremental task to validate a set of JSON files against against a schema.
+ * Incremental task to validate a set of JSON files against a schema.
  */
 public class ValidateJsonAgainstSchemaTask extends DefaultTask {
     private File jsonSchema;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java

@@ -12,7 +12,7 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
 /**
- * Incremental task to validate a set of YAML files against against a schema.
+ * Incremental task to validate a set of YAML files against a schema.
  */
 public class ValidateYamlAgainstSchemaTask extends ValidateJsonAgainstSchemaTask {
     @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java

@@ -11,148 +11,72 @@
 import com.google.common.annotations.VisibleForTesting;
 
 import org.elasticsearch.gradle.VersionProperties;
-import org.gradle.api.GradleException;
 
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
 
 import static java.util.Comparator.comparing;
 import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
 
 /**
- * Generates the page that contains an index into the breaking changes and lists deprecations for a minor version release,
- * and the individual pages for each breaking area.
+ * Generates the page that contains breaking changes deprecations for a minor release series.
  */
 public class BreakingChangesGenerator {
 
-    // Needs to match `changelog-schema.json`
-    private static final List<String> BREAKING_AREAS = List.of(
-        "Cluster and node setting",
-        "Command line tool",
-        "Index setting",
-        "JVM option",
-        "Java API",
-        "Logging",
-        "Mapping",
-        "Packaging",
-        "Painless",
-        "REST API",
-        "System requirement",
-        "Transform"
-    );
-
-    static void update(
-        File indexTemplateFile,
-        File indexOutputFile,
-        File outputDirectory,
-        File areaTemplateFile,
-        List<ChangelogEntry> entries
-    ) throws IOException {
-        if (outputDirectory.exists()) {
-            if (outputDirectory.isDirectory() == false) {
-                throw new GradleException("Path [" + outputDirectory + "] exists but isn't a directory!");
-            }
-        } else {
-            Files.createDirectory(outputDirectory.toPath());
-        }
-
-        try (FileWriter output = new FileWriter(indexOutputFile)) {
+    static void update(File migrationTemplateFile, File migrationOutputFile, List<ChangelogEntry> entries) throws IOException {
+        try (FileWriter output = new FileWriter(migrationOutputFile)) {
             output.write(
-                generateIndexFile(
+                generateMigrationFile(
                     QualifiedVersion.of(VersionProperties.getElasticsearch()),
-                    Files.readString(indexTemplateFile.toPath()),
+                    Files.readString(migrationTemplateFile.toPath()),
                     entries
                 )
             );
         }
-
-        String areaTemplate = Files.readString(areaTemplateFile.toPath());
-
-        for (String breakingArea : BREAKING_AREAS) {
-            final List<ChangelogEntry.Breaking> entriesForArea = entries.stream()
-                .map(ChangelogEntry::getBreaking)
-                .filter(entry -> entry != null && breakingArea.equals(entry.getArea()))
-                .collect(Collectors.toList());
-
-            if (entriesForArea.isEmpty()) {
-                continue;
-            }
-
-            final String outputFilename = breakingArea.toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "-")
-                + "-changes.asciidoc";
-
-            try (FileWriter output = new FileWriter(outputDirectory.toPath().resolve(outputFilename).toFile())) {
-                output.write(
-                    generateBreakingAreaFile(
-                        QualifiedVersion.of(VersionProperties.getElasticsearch()),
-                        areaTemplate,
-                        breakingArea,
-                        entriesForArea
-                    )
-                );
-            }
-        }
     }
 
     @VisibleForTesting
-    static String generateIndexFile(QualifiedVersion version, String template, List<ChangelogEntry> entries) throws IOException {
-        final Map<String, List<ChangelogEntry.Deprecation>> deprecationsByArea = entries.stream()
+    static String generateMigrationFile(QualifiedVersion version, String template, List<ChangelogEntry> entries) throws IOException {
+        final Map<Boolean, Map<String, List<ChangelogEntry.Deprecation>>> deprecationsByNotabilityByArea = entries.stream()
             .map(ChangelogEntry::getDeprecation)
             .filter(Objects::nonNull)
             .sorted(comparing(ChangelogEntry.Deprecation::getTitle))
-            .collect(groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, Collectors.toList()));
-
-        final List<String> breakingIncludeList = entries.stream()
-            .filter(each -> each.getBreaking() != null)
-            .map(each -> each.getBreaking().getArea().toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "-"))
-            .distinct()
-            .sorted()
-            .toList();
-
-        final Map<String, Object> bindings = new HashMap<>();
-        bindings.put("breakingIncludeList", breakingIncludeList);
-        bindings.put("deprecationsByArea", deprecationsByArea);
-        bindings.put("isElasticsearchSnapshot", version.isSnapshot());
-        bindings.put("majorDotMinor", version.major() + "." + version.minor());
-        bindings.put("majorMinor", String.valueOf(version.major()) + version.minor());
-        bindings.put("nextMajor", (version.major() + 1) + ".0");
-        bindings.put("version", version);
-
-        return TemplateUtils.render(template, bindings);
-    }
+            .collect(
+                groupingBy(
+                    ChangelogEntry.Deprecation::isNotable,
+                    TreeMap::new,
+                    groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, toList())
+                )
+            );
 
-    @VisibleForTesting
-    static String generateBreakingAreaFile(
-        QualifiedVersion version,
-        String template,
-        String breakingArea,
-        List<ChangelogEntry.Breaking> entriesForArea
-    ) throws IOException {
-        final Map<Boolean, Set<ChangelogEntry.Breaking>> breakingEntriesByNotability = entriesForArea.stream()
+        final Map<Boolean, Map<String, List<ChangelogEntry.Breaking>>> breakingByNotabilityByArea = entries.stream()
+            .map(ChangelogEntry::getBreaking)
+            .filter(Objects::nonNull)
+            .sorted(comparing(ChangelogEntry.Breaking::getTitle))
             .collect(
                 groupingBy(
                     ChangelogEntry.Breaking::isNotable,
-                    toCollection(() -> new TreeSet<>(comparing(ChangelogEntry.Breaking::getTitle)))
+                    TreeMap::new,
+                    groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, toList())
                 )
             );
 
         final Map<String, Object> bindings = new HashMap<>();
-        bindings.put("breakingArea", breakingArea);
-        bindings.put("breakingEntriesByNotability", breakingEntriesByNotability);
-        bindings.put("breakingAreaAnchor", breakingArea.toLowerCase(Locale.ROOT).replaceFirst(" and", "").replaceAll(" ", "_"));
+        bindings.put("breakingByNotabilityByArea", breakingByNotabilityByArea);
+        bindings.put("deprecationsByNotabilityByArea", deprecationsByNotabilityByArea);
+        bindings.put("isElasticsearchSnapshot", version.isSnapshot());
+        bindings.put("majorDotMinor", version.major() + "." + version.minor());
         bindings.put("majorMinor", String.valueOf(version.major()) + version.minor());
+        bindings.put("nextMajor", (version.major() + 1) + ".0");
+        bindings.put("version", version);
 
         return TemplateUtils.render(template, bindings);
     }