Fix get-license test failure by ensure cluster is ready (#60498) (#60569)

When a new cluster starts, the HTTP layer becomes ready to accept incoming 
requests while the basic license is still being populated in the background. 
When a get license request comes in before the license is ready, it can get 
404 error. This PR fixes it by either wrap the license check in assertBusy or 
ensure the license is ready before perform the check.

This is a backport for both #60498 and #60573

Author: ywangd

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -104,7 +104,7 @@ protected ClientYamlTestClient initClientYamlTestClient(
 
     @Before
     public void waitForRequirements() throws Exception {
-        if (isCcrTest()) {
+        if (isCcrTest() || isGetLicenseTest()) {
             ESRestTestCase.waitForActiveLicense(adminClient());
         }
     }
@@ -175,6 +175,11 @@ protected boolean isCcrTest() {
         return testName != null && testName.contains("/ccr/");
     }
 
+    protected boolean isGetLicenseTest() {
+        String testName = getTestName();
+        return testName != null && (testName.contains("/get-license/") || testName.contains("\\get-license\\"));
+    }
+
     /**
      * Compares the results of running two analyzers against many random
      * strings. The goal is to figure out if two anlayzers are "the same" by

x-pack/plugin/security/qa/basic-enable-security/src/test/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java

@@ -97,10 +97,16 @@ private String getIndices() throws IOException {
         return EntityUtils.toString(response.getEntity());
     }
 
-    private void checkBasicLicenseType() throws IOException {
-        Map<String, Object> license = getAsMap("/_license");
-        assertThat(license, notNullValue());
-        assertThat(ObjectPath.evaluate(license, "license.type"), equalTo("basic"));
+    private void checkBasicLicenseType() throws Exception {
+        assertBusy(() -> {
+            try {
+                Map<String, Object> license = getAsMap("/_license");
+                assertThat(license, notNullValue());
+                assertThat(ObjectPath.evaluate(license, "license.type"), equalTo("basic"));
+            } catch (ResponseException e) {
+                throw new AssertionError(e);
+            }
+        });
     }
 
     private void checkSecurityStatus(boolean expectEnabled) throws IOException {

x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java

@@ -77,10 +77,16 @@ private void revertTrial() throws IOException {
         client().performRequest(new Request("POST", "/_license/start_basic?acknowledge=true"));
     }
 
-    private void checkLicenseType(String type) throws IOException {
-        Map<String, Object> license = getAsMap("/_license");
-        assertThat(license, notNullValue());
-        assertThat(ObjectPath.evaluate(license, "license.type"), equalTo(type));
+    private void checkLicenseType(String type) throws Exception {
+        assertBusy(() -> {
+            try {
+                Map<String, Object> license = getAsMap("/_license");
+                assertThat(license, notNullValue());
+                assertThat(ObjectPath.evaluate(license, "license.type"), equalTo(type));
+            } catch (ResponseException e) {
+                throw new AssertionError(e);
+            }
+        });
     }
 
     private void checkSecurityEnabled(boolean allowAllRealms) throws IOException {

x-pack/plugin/security/qa/tls-basic/src/test/java/org/elasticsearch/xpack/security/TlsWithBasicLicenseIT.java

@@ -7,6 +7,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -85,10 +86,17 @@ private void revertTrial() throws IOException {
         client().performRequest(new Request("POST", "/_license/start_basic?acknowledge=true"));
     }
 
-    private void checkLicenseType(String type) throws IOException {
-        Map<String, Object> license = getAsMap("/_license");
-        assertThat(license, notNullValue());
-        assertThat(ObjectPath.evaluate(license, "license.type"), equalTo(type));
+    private void checkLicenseType(String type) throws Exception {
+        assertBusy(() -> {
+            try {
+                Map<String, Object> license = getAsMap("/_license");
+                assertThat(license, notNullValue());
+                assertThat(ObjectPath.evaluate(license, "license.type"), equalTo(type));
+            } catch (ResponseException e) {
+                throw new AssertionError(e);
+            }
+        });
+
     }
 
     private void checkSSLEnabled() throws IOException {