Add build utility to check cluster health over ssl

buildSrc/build.gradle

@@ -240,6 +240,7 @@ if (project != rootProject) {
 
   forbiddenPatterns {
     exclude '**/*.wav'
+    exclude '**/*.p12'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
   }

buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java

@@ -0,0 +1,233 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.http;
+
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
+ * This is logically similar to using the Ant Get task to retrieve a resource, but with the difference that it can
+ * access resources that do not use the JRE's default trusted CAs.
+ */
+public class WaitForHttpResource {
+
+    private static final Logger logger = Logging.getLogger(WaitForHttpResource.class);
+
+    private Set<Integer> validResponseCodes = Collections.singleton(200);
+    private URL url;
+    private Set<File> certificateAuthorities;
+    private File trustStoreFile;
+    private String trustStorePassword;
+    private String username;
+    private String password;
+
+    public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
+        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
+    }
+
+    public WaitForHttpResource(URL url) {
+        this.url = url;
+    }
+
+    public void setValidResponseCodes(int... validResponseCodes) {
+        this.validResponseCodes = new HashSet<>(validResponseCodes.length);
+        for (int rc : validResponseCodes) {
+            this.validResponseCodes.add(rc);
+        }
+    }
+
+    public void setCertificateAuthorities(File... certificateAuthorities) {
+        this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
+    }
+
+    public void setTrustStoreFile(File trustStoreFile) {
+        this.trustStoreFile = trustStoreFile;
+    }
+
+    public void setTrustStorePassword(String trustStorePassword) {
+        this.trustStorePassword = trustStorePassword;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
+        final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
+        final long sleep = Long.max(durationInMs / 10, 100);
+
+        final SSLContext ssl;
+        final KeyStore trustStore = buildTrustStore();
+        if (trustStore != null) {
+            ssl = createSslContext(trustStore);
+        } else {
+            ssl = null;
+        }
+        IOException failure = null;
+        for (; ; ) {
+            try {
+                checkResource(ssl);
+                return true;
+            } catch (IOException e) {
+                logger.debug("Failed to access resource [{}]", url, e);
+                failure = e;
+            }
+            if (System.nanoTime() < waitUntil) {
+                Thread.sleep(sleep);
+            } else {
+                logger.error("Failed to access url [{}]", url, failure);
+                return false;
+            }
+        }
+    }
+
+    protected void checkResource(SSLContext ssl) throws IOException {
+        try {
+            final HttpURLConnection connection = buildConnection(ssl);
+            connection.connect();
+            final Integer response = connection.getResponseCode();
+            if (validResponseCodes.contains(response)) {
+                logger.info("Got successful response [{}] from URL [{}]", response, url);
+                return;
+            } else {
+                throw new IOException(response + " " + connection.getResponseMessage());
+            }
+        } catch (IOException e) {
+            throw e;
+        }
+    }
+
+    HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
+        final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
+        configureSslContext(connection, ssl);
+        configureBasicAuth(connection);
+        connection.setRequestMethod("GET");
+        return connection;
+    }
+
+    private void configureSslContext(HttpURLConnection connection, SSLContext ssl) {
+        if (ssl != null) {
+            if (connection instanceof HttpsURLConnection) {
+                ((HttpsURLConnection) connection).setSSLSocketFactory(ssl.getSocketFactory());
+            } else {
+                throw new IllegalStateException("SSL trust has been configured, but [" + url + "] is not a 'https' URL");
+            }
+        }
+    }
+
+    private void configureBasicAuth(HttpURLConnection connection) {
+        if (username != null) {
+            if (password == null) {
+                throw new IllegalStateException("Basic Auth user [" + username
+                    + "] has been set, but no password has been configured");
+            }
+            connection.setRequestProperty("Authorization",
+                "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)));
+        }
+    }
+
+    KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
+        if (this.certificateAuthorities != null) {
+            if (trustStoreFile != null) {
+                throw new IllegalStateException("Cannot specify both truststore and CAs");
+            }
+            return buildTrustStoreFromCA();
+        } else if (trustStoreFile != null) {
+            return buildTrustStoreFromFile();
+        } else {
+            return null;
+        }
+    }
+
+    private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
+        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        try (InputStream input = new FileInputStream(trustStoreFile)) {
+            keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
+        }
+        return keyStore;
+    }
+
+    private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
+        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        store.load(null, null);
+        final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        int counter = 0;
+        for (File ca : certificateAuthorities) {
+            try (InputStream input = new FileInputStream(ca)) {
+                for (Certificate certificate : certFactory.generateCertificates(input)) {
+                    store.setCertificateEntry("cert-" + counter, certificate);
+                    counter++;
+                }
+            }
+        }
+        return store;
+    }
+
+    private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
+        checkForTrustEntry(trustStore);
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(trustStore);
+        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
+        return sslContext;
+    }
+
+    private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
+        Enumeration<String> enumeration = trustStore.aliases();
+        while (enumeration.hasMoreElements()) {
+            if (trustStore.isCertificateEntry(enumeration.nextElement())) {
+                // found trusted cert entry
+                return;
+            }
+        }
+        throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
+    }
+}

buildSrc/src/test/java/org/elasticsearch/gradle/http/WaitForHttpResourceTests.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.http;
+
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+
+import java.io.File;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+public class WaitForHttpResourceTests extends GradleUnitTestCase {
+
+    public void testBuildTrustStoreFromFile() throws Exception {
+        final WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"));
+        final URL ca = getClass().getResource("/ca.p12");
+        assertThat(ca, notNullValue());
+        http.setTrustStoreFile(new File(ca.getPath()));
+        http.setTrustStorePassword("password");
+        final KeyStore store = http.buildTrustStore();
+        final Certificate certificate = store.getCertificate("ca");
+        assertThat(certificate, notNullValue());
+        assertThat(certificate, instanceOf(X509Certificate.class));
+        assertThat(((X509Certificate)certificate).getSubjectDN().toString(), equalTo("CN=Elastic Certificate Tool Autogenerated CA"));
+    }
+
+    public void testBuildTrustStoreFromCA() throws Exception {
+        final WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"));
+        final URL ca = getClass().getResource("/ca.pem");
+        assertThat(ca, notNullValue());
+        http.setCertificateAuthorities(new File(ca.getPath()));
+        final KeyStore store = http.buildTrustStore();
+        final Certificate certificate = store.getCertificate("cert-0");
+        assertThat(certificate, notNullValue());
+        assertThat(certificate, instanceOf(X509Certificate.class));
+        assertThat(((X509Certificate)certificate).getSubjectDN().toString(), equalTo("CN=Elastic Certificate Tool Autogenerated CA"));
+    }
+}

buildSrc/src/test/resources/ca.pem

@@ -0,0 +1,25 @@
+Bag Attributes
+    friendlyName: ca
+    localKeyID: 54 69 6D 65 20 31 35 35 33 37 34 33 38 39 30 38 33 35 
+subject=/CN=Elastic Certificate Tool Autogenerated CA
+issuer=/CN=Elastic Certificate Tool Autogenerated CA
+-----BEGIN CERTIFICATE-----
+MIIDSjCCAjKgAwIBAgIVAMQMmDRcXfXLaTp6ep1H8rC3tOrwMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTE5MDMyODAzMzEyNloXDTIyMDMyNzAzMzEyNlowNDEyMDAG
+A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDT73N6JZeBPyzahc0aNcra
+BpUROVGB9wXQqf8JeU4GtH+1qfqUKYKUJTe/DZWc+5Qz1WAKGZEvBySAlgbuncuq
+VpLzWxpEui1vRW8JB3gjZgeY3vfErrEWWr95YM0e8rWu4AoAchzqsrG0/+po2eui
+cN+8hI6jRKiBv/ZeQqja6KZ8y4Wt4VaNVL53+I7+eWA/aposu6/piUg2wZ/FNhVK
+hypcJwDdp3fQaugtPj3y76303jTRgutgd3rtWFuy3MCDLfs3mSQUjO10s93zwLdC
+XokyIywijS5CpO8mEuDRu9rb5J1DzwUpUfk+GMObb6rHjFKzSqnM3s+nasypQQ9L
+AgMBAAGjUzBRMB0GA1UdDgQWBBQZEW88R95zSzO2tLseEWgI7ugvLzAfBgNVHSME
+GDAWgBQZEW88R95zSzO2tLseEWgI7ugvLzAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQBEJN0UbL77usVnzIvxKa3GpLBgJQAZtD1ifZppC4w46Bul
+1G7Fdc+XMbzZlI4K6cWEdd5dfEssKA8btEtRzdNOqgggBpqrUU0mNlQ+vC22XORU
+ykHAu2TsRwoHmuxkd9Et/QyuTFXR4fTiU8rsJuLFOgn+RdEblA0J0gJeIqdWI5Z1
+z13OyZEl6BCQFyrntu2eERxaHEfsJOSBZE4RcecnLNGhIJBXE0Pk4iTiViJF/h7d
++kUUegKx0qewZif2eEZgrz12Vuen9a6bh2i2pNS95vABVVMr8uB+J1BGkNA5YT7J
+qtZA2tN//Evng7YDiR+KkB1kvXVZVIi2WPDLD/zu
+-----END CERTIFICATE-----

x-pack/qa/reindex-tests-with-security/build.gradle

@@ -1,10 +1,4 @@
-import javax.net.ssl.HttpsURLConnection
-import javax.net.ssl.KeyManager
-import javax.net.ssl.SSLContext
-import javax.net.ssl.TrustManagerFactory
-import java.nio.charset.StandardCharsets
-import java.security.KeyStore
-import java.security.SecureRandom
+import org.elasticsearch.gradle.http.WaitForHttpResource
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
@@ -57,48 +51,11 @@ integTestCluster {
                  'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role
   }
   waitCondition = { node, ant ->
-    // Load the CA PKCS#12 file as a truststore
-    KeyStore ks = KeyStore.getInstance("PKCS12");
-    ks.load(caFile.newInputStream(), 'password'.toCharArray());
-    TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-    tmf.init(ks);
-
-    // Configre a SSL context for TLS1.2 using our CA trust manager
-    SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-    sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
-
-    // Check whether the cluster has started
-    URL url = new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow");
-    for (int i = 20; i >= 0; i--) {
-      // we use custom wait logic here for HTTPS
-      HttpsURLConnection httpURLConnection = null;
-      try {
-        logger.info("Trying ${url}");
-        httpURLConnection = (HttpsURLConnection) url.openConnection();
-        httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory());
-        httpURLConnection.setRequestProperty("Authorization",
-          "Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
-        httpURLConnection.setRequestMethod("GET");
-        httpURLConnection.connect();
-        if (httpURLConnection.getResponseCode() == 200) {
-          logger.info("Cluster has started");
-          return true;
-        } else {
-          logger.debug("HTTP response was [{}]", httpURLConnection.getResponseCode());
-        }
-      } catch (IOException e) {
-          if (i == 0) {
-              logger.error("Failed to call cluster health - " + e)
-          }
-          logger.debug("Call to [{}] threw an exception", url, e)
-      } finally {
-        if (httpURLConnection != null) {
-          httpURLConnection.disconnect();
-        }
-      }
-      // did not start, so wait a bit before trying again
-      Thread.sleep(750L);
-    }
-    return false;
+    WaitForHttpResource http = new WaitForHttpResource("https", node.httpUri(), numNodes)
+    http.setTrustStoreFile(caFile)
+    http.setTrustStorePassword("password")
+    http.setUsername("test_admin")
+    http.setPassword("x-pack-test-password")
+    return http.wait(5000)
   }
 }