Merge branch '7.x' into docker-from-artifacts-7.x

* 7.x:
  Make qa/full-cluster-restart tests pass. By fixing a helper method and (elastic#38604)
  Mute failing WatchStatusIntegrationTests (elastic#38621)
  Mute failing  ApiKeyIntegTests (elastic#38614)
  [DOCS] Add warning about bypassing ML PUT APIs (elastic#38605)
  Mute RetentionLeastIT.testRetentionLeasesSyncOnRecovery on 7x (elastic#38597)
  Only "include_type_name" if running on >= 7 (elastic#38594)
  Fix version logic when bumping major version (elastic#38593)

Author: jasontedor

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -218,7 +218,14 @@ private String getGradleProjectNameFor(Version version) {
     private String getBranchFor(Version version) {
         switch (getGradleProjectNameFor(version)) {
             case "minor":
-                return version.getMajor() + ".x";
+                // The .x branch will always point to the latest minor (for that major), so a "minor" project will be on the .x branch
+                //  unless there is more recent (higher) minor.
+                final Version latestInMajor = getLatestVersionByKey(groupByMajor, version.getMajor());
+                if (latestInMajor.getMinor() == version.getMinor()) {
+                    return version.getMajor() + ".x";
+                } else {
+                    return version.getMajor() + "." + version.getMinor();
+                }
             case "staged":
             case "maintenance":
             case "bugfix":
@@ -234,7 +241,15 @@ public List<Version> getUnreleased() {
         unreleased.add(currentVersion);
 
         // the tip of the previous major is unreleased for sure, be it a minor or a bugfix
-        unreleased.add(getLatestVersionByKey(this.groupByMajor, currentVersion.getMajor() - 1));
+        final Version latestOfPreviousMajor = getLatestVersionByKey(this.groupByMajor, currentVersion.getMajor() - 1);
+        unreleased.add(latestOfPreviousMajor);
+        if (latestOfPreviousMajor.getRevision() == 0) {
+            // if the previous major is a x.y.0 release, then the tip of the minor before that (y-1) is also unreleased
+            final Version previousMinor = getLatestInMinor(latestOfPreviousMajor.getMajor(), latestOfPreviousMajor.getMinor() - 1);
+            if (previousMinor != null) {
+                unreleased.add(previousMinor);
+            }
+        }
 
         final Map<Integer, List<Version>> groupByMinor = getReleasedMajorGroupedByMinor();
         int greatestMinor = groupByMinor.keySet().stream().max(Integer::compareTo).orElse(0);
@@ -262,6 +277,13 @@ public List<Version> getUnreleased() {
         );
     }
 
+    private Version getLatestInMinor(int major, int minor) {
+        return groupByMajor.get(major).stream()
+            .filter(v -> v.getMinor() == minor)
+            .max(Version::compareTo)
+            .orElse(null);
+    }
+
     private Version getLatestVersionByKey(Map<Integer, List<Version>> groupByMajor, int key) {
         return groupByMajor.getOrDefault(key, emptyList()).stream()
             .max(Version::compareTo)

buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java

@@ -282,6 +282,10 @@ public void testGetUnreleased() {
             asList("7.1.1", "7.2.0", "7.3.0", "8.0.0"),
             getVersionCollection("8.0.0").getUnreleased()
         );
+        assertVersionsEquals(
+            asList("6.6.1", "6.7.0", "7.0.0", "7.1.0"),
+            getVersionCollection("7.1.0").getUnreleased()
+        );
     }
 
     public void testGetBranch() {
@@ -298,13 +302,17 @@ public void testGetBranch() {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedBranchNames(
-            asList("5.6", "6.4", "6.x"),
+            asList("5.6", "6.4", "6.5"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedBranchNames(
             asList("7.1", "7.2", "7.x"),
             getVersionCollection("8.0.0")
         );
+        assertUnreleasedBranchNames(
+            asList("6.6", "6.7", "7.0"),
+            getVersionCollection("7.1.0")
+        );
     }
 
     public void testGetGradleProjectName() {
@@ -329,7 +337,7 @@ public void testGetGradleProjectName() {
             getVersionCollection("8.0.0")
         );
         assertUnreleasedGradleProjectNames(
-            asList("staged", "minor"),
+            asList("maintenance", "staged", "minor"),
             getVersionCollection("7.1.0")
         );
     }

docs/reference/ml/apis/put-datafeed.asciidoc

@@ -19,6 +19,11 @@ Instantiates a {dfeed}.
 You must create a job before you create a {dfeed}.  You can associate only one
 {dfeed} to each job.
 
+IMPORTANT:  You must use {kib} or this API to create a {dfeed}. Do not put a {dfeed}
+            directly to the `.ml-config` index using the Elasticsearch index API.
+            If {es} {security-features} are enabled, do not give users `write`
+            privileges on the `.ml-config` index.
+
 
 ==== Path Parameters
 

docs/reference/ml/apis/put-job.asciidoc

@@ -12,7 +12,13 @@ Instantiates a job.
 
 `PUT _ml/anomaly_detectors/<job_id>`
 
-//===== Description
+===== Description
+
+IMPORTANT: You must use {kib} or this API to create a {ml} job. Do not put a job
+            directly to the `.ml-config` index using the Elasticsearch index API.
+            If {es} {security-features} are enabled, do not give users `write`
+            privileges on the `.ml-config` index.
+
 
 ==== Path Parameters
 

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -843,6 +843,7 @@ public void testRecovery() throws Exception {
      * old and new versions. All of the snapshots include an index, a template,
      * and some routing configuration.
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38603")
     public void testSnapshotRestore() throws IOException {
         int count;
         if (isRunningAgainstOldCluster() && getOldClusterVersion().major < 8) {
@@ -909,8 +910,8 @@ public void testSnapshotRestore() throws IOException {
         createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
 
         // In 7.0, type names are no longer expected by default in put index template requests.
-        // We therefore use the deprecated typed APIs when running against the current version.
-        if (isRunningAgainstAncientCluster()) {
+        // We therefore use the deprecated typed APIs when running against the current version, but testing with a pre-7 version
+        if (isRunningAgainstOldCluster() == false && getOldClusterVersion().major < 7) {
             createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true");
         }
         createTemplateRequest.setOptions(allowTypeRemovalWarnings());
@@ -1214,7 +1215,7 @@ private void saveInfoDocument(String type, String value) throws IOException {
     private String loadInfoDocument(String type) throws IOException {
         Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type);
         request.addParameter("filter_path", "_source");
-        if (isRunningAgainstAncientCluster()) {
+        if (getOldClusterVersion().before(Version.V_6_7_0)) {
             request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
         }
         String doc = toStr(client().performRequest(request));

server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java

@@ -260,6 +260,7 @@ public void testBackgroundRetentionLeaseSync() throws Exception {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38588")
     public void testRetentionLeasesSyncOnRecovery() throws Exception {
         final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2);
         internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas);

test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java

@@ -43,7 +43,7 @@ public class VersionUtils {
      * rules here match up with the rules in gradle then this should
      * produce sensible results.
      * @return a tuple containing versions with backwards compatibility
-     * guarantees in v1 and versions without the guranteees in v2
+     * guarantees in v1 and versions without the guarantees in v2
      */
     static Tuple<List<Version>, List<Version>> resolveReleasedVersions(Version current, Class<?> versionClass) {
         // group versions into major version
@@ -67,7 +67,11 @@ static Tuple<List<Version>, List<Version>> resolveReleasedVersions(Version curre
             // on a stable or release branch, ie N.x
             stableVersions = currentMajor;
             // remove the next maintenance bugfix
-            moveLastToUnreleased(previousMajor, unreleasedVersions);
+            final Version prevMajorLastMinor = moveLastToUnreleased(previousMajor, unreleasedVersions);
+            if (prevMajorLastMinor.revision == 0 && previousMajor.isEmpty() == false) {
+                // The latest minor in the previous major is a ".0" release, so there must be an unreleased bugfix for the minor before that
+                moveLastToUnreleased(previousMajor, unreleasedVersions);
+            }
         }
 
         // remove next minor

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java

@@ -237,6 +237,7 @@ public void testInvalidateApiKeysForApiKeyName() throws InterruptedException, Ex
         verifyInvalidateResponse(1, responses, invalidateResponse);
     }
 
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/38408")
     public void testGetAndInvalidateApiKeysWithExpiredAndInvalidatedApiKey() throws Exception {
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Instant created = Instant.now();

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusIntegrationTests.java

@@ -24,6 +24,7 @@
 
 public class WatchStatusIntegrationTests extends AbstractWatcherIntegrationTestCase {
 
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/38619")
     public void testThatStatusGetsUpdated() {
         WatcherClient watcherClient = watcherClient();
         watcherClient.preparePutWatch("_name")