Add HTTPS support to AzureHttpFixture

modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java

@@ -46,7 +46,7 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi
 
     @ClassRule
     public static AzureHttpFixture fixture = new AzureHttpFixture(
-        USE_FIXTURE,
+        USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
         System.getProperty("test.azure.account"),
         System.getProperty("test.azure.container")
     );

modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java

@@ -14,6 +14,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
@@ -28,7 +29,15 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER
+    );
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
@@ -45,14 +54,15 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
         )
         .setting(
             "azure.client.integration_test.endpoint_suffix",
-            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
+            () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(),
             s -> USE_FIXTURE
         )
         .setting("thread_pool.repository_azure.max", () -> String.valueOf(randomIntBetween(1, 10)), s -> USE_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE)
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);

test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java

@@ -8,42 +8,116 @@
 package fixture.azure;
 
 import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
 
+import org.elasticsearch.common.ssl.KeyStoreUtil;
+import org.elasticsearch.common.ssl.PemUtils;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.rules.ExternalResource;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.util.List;
+import java.util.Objects;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+
+import static org.elasticsearch.test.ESTestCase.assertThat;
+import static org.elasticsearch.test.ESTestCase.fail;
+import static org.hamcrest.Matchers.hasSize;
 
 public class AzureHttpFixture extends ExternalResource {
 
-    private final boolean enabled;
+    private final Protocol protocol;
     private final String account;
     private final String container;
     private HttpServer server;
 
-    public AzureHttpFixture(boolean enabled, String account, String container) {
-        this.enabled = enabled;
+    public enum Protocol {
+        NONE,
+        HTTP,
+        HTTPS
+    }
+
+    public AzureHttpFixture(Protocol protocol, String account, String container) {
+        this.protocol = protocol;
         this.account = account;
         this.container = container;
     }
 
+    private String scheme() {
+        return switch (protocol) {
+            case NONE -> fail(null, "fixture is disabled");
+            case HTTP -> "http";
+            case HTTPS -> "https";
+        };
+    }
+
     public String getAddress() {
-        return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account;
+        return scheme() + "://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account;
     }
 
     @Override
-    protected void before() throws IOException {
-        if (enabled) {
-            this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
-            server.createContext("/" + account, new AzureHttpHandler(account, container));
-            server.start();
+    protected void before() {
+        try {
+            switch (protocol) {
+                case NONE -> {
+                }
+                case HTTP -> {
+                    server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                    server.createContext("/" + account, new AzureHttpHandler(account, container));
+                    server.start();
+                }
+                case HTTPS -> {
+                    final var httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                    this.server = httpsServer;
+                    final var tmpdir = ESTestCase.createTempDir();
+                    final var certificates = PemUtils.readCertificates(List.of(copyResource(tmpdir, "azure-http-fixture.pem")));
+                    assertThat(certificates, hasSize(1));
+                    final SSLContext sslContext = SSLContext.getInstance("TLS");
+                    sslContext.init(
+                        new KeyManager[] {
+                            KeyStoreUtil.createKeyManager(
+                                new Certificate[] { certificates.get(0) },
+                                PemUtils.readPrivateKey(copyResource(tmpdir, "azure-http-fixture.key"), () -> null),
+                                null
+                            ) },
+                        null,
+                        new SecureRandom()
+                    );
+                    httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
+                    httpsServer.createContext("/" + account, new AzureHttpHandler(account, container));
+                    httpsServer.start();
+                }
+            }
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+
+    private Path copyResource(Path tmpdir, String name) throws IOException {
+        try (
+            var stream = Objects.requireNonNullElseGet(
+                getClass().getResourceAsStream(name),
+                () -> ESTestCase.fail(null, "resource [%s] not found", name)
+            )
+        ) {
+            final var path = tmpdir.resolve(name);
+            Files.write(path, stream.readAllBytes());
+            return path;
         }
     }
 
     @Override
     protected void after() {
-        if (enabled) {
+        if (protocol != Protocol.NONE) {
             server.stop(0);
         }
     }

test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCswbyZDtaghZXs
+Phs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+lTMhKOUpHuphgC31qbf/G
+vQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHazTYwIlzVs4sFPm5i2fQb
+DK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQHzLZlfYkkM2bCaXBgKWV
+wmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdkK8qJ0Lpchg3zlP4qIzbm
+WRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA8qnqtKYTqMzo3uv1ESlE
+R8EAZDUbAgMBAAECggEAY8lYRdSTVo8y5Q2OrCQa6b38jvC2cfKY4enMbir4JZKT
+lllzA7VtEUGpgzKRsoXbCQmYAEpCvBojlskQe4KJgW50gxVjaQa9zVhM55vhbdzc
+AJaOWD0CUjRsSbUlKrJ+ixW1JGdGXaTlYkZ2K0AalLT/N1Y8RKN4FWmEyKCvcvz4
+0XzOIVmG+HqcNURamXTxMKbj1yzi5goue2/iP2kMFo8sHxRsGvvV4PWo6JrktE2y
+47oiH42lpSIcpLSE8z/csLbMTw/Q/pPQAYqyvEJHU22uhac1XmMqFHWNSpQZq6gr
+46t8YQ3FJSN8UrZf1h1mdvLlK/sgPEvCQa6TrCq4GQKBgQDbl0M/4gJZhgpvBuCC
+aratamWcFoa/pu1/JoQhPXLv6uGwB/cFhJUVr0ZoI5KPFJr9SG4kQ/eEPywkY2qT
+mGPVlVmGOeJa1VK8TRUTzkEFTYWytUepACM2//LiWvzABciO8SxWgNZrmUKghQTN
+d989b8edy0ti6y7lHpkTyawVXQKBgQDJZo7X6K+6cyuXORApv98cU5C3vEVtBp/c
+QfU/rRj/YXAbYFhKIS5rF/Gfg2YoDa5YguMxGsIPzYwdTI5gGGgSgolapz3fr22q
+edCPaFg8qO64pIii+Ar4lx4k1IyNtpJ+nvlam7sI9yGzksrVazsWhpaSKX8xGd7r
+9ZSr/c8U1wKBgGWl+pJay52nR7MnasvUHCXgR5LedpfG7M9cA/PjHw5iGwDCXx2l
+xuFX1m6kcNZcwnYWji2pbK1CFOvvPUl/VE9tKBjTOK21a+wQfn5BjqWmwgn8kmRv
+1N1D06nmVnOI+dL5Xv3X++mo80ec66E1KRimYq/viEEM/xM+e7vGMitdAoGAUAUe
+pix+fa8615fFk0D37bJKIqZ8Uyg5pfLS9ZzZ/MYDG+14xuNOJSDbUMyNb0aYSfSf
+PihqiIrbq9x6CTZJS2lwF4Oxcsmp4f0KX6BOxrM8PkKpQ08YVNL+GBYXTksHA6Y4
+XsbXVmWSj124l3lGfdm1w5cXQTQNPWVSz89FUvsCgYEArl27gbXndTLpZ4dNkeBS
+0JMe84nPPrrwHbNwcqkoDiux5Ln+AZE8jYSlp4kUfd1A7XDTxXWXIxeD0YLiabf3
+5+/QzJ6j1qi77JoyadomnL7CFI5f2FotKt029PeVxAUOohao94p8J5OuaRMPvkGC
+CNhjfRAIBhfm9kdyjPmwVLU=
+-----END PRIVATE KEY-----

test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRTCCAi2gAwIBAgIVAJpxxIbXWyvdd6/rIFXPgWe6fyvTMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTE4MTIyMTA3NDY1NVoXDTQ2MDUwNzA3NDY1NVowGTEXMBUG
+A1UEAxMObGRhcC10ZXN0LWNhc2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCswbyZDtaghZXsPhs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+l
+TMhKOUpHuphgC31qbf/GvQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHa
+zTYwIlzVs4sFPm5i2fQbDK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQ
+HzLZlfYkkM2bCaXBgKWVwmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdk
+K8qJ0Lpchg3zlP4qIzbmWRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA
+8qnqtKYTqMzo3uv1ESlER8EAZDUbAgMBAAGjaTBnMB0GA1UdDgQWBBQaiCDScfBa
+jHOSk04XOymffbLBxTAfBgNVHSMEGDAWgBROJaHRWe17um5rqqYn10aqedr55DAa
+BgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCQYDVR0TBAIwADANBgkqhkiG9w0B
+AQsFAAOCAQEAXBovNqVg+VQ1LR0PfEMpbgbQlekky8qY2y1tz7J0ntGepAq+Np6n
+7J9En6ty1ELZUvgPUCF2btQqZbv8uyHz/C+rojKC5xzHN5qbZ31o5/0I/kNase1Z
+NbXuNJe3wAXuz+Mj5rtuOGZvlFsbtocuoydVYOclfqjUXcoZtqCcRamSvye7vGl2
+CHPqDi0uK8d75nE9Jrnmz/BNNV7CjPg636PJmCUrLL21+t69ZFL1eGAFtLBmmjcw
+cMkyv9bJirjZbjt/9UB+fW9XzV3RVLAzfrIHtToupXmWc4+hTOnlbKfFwqB9fa7Y
+XcCfGrZoJg9di1HbJrSJmv5QgRTM+/zkrA==
+-----END CERTIFICATE-----

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -155,6 +155,7 @@
 import java.lang.invoke.MethodHandles;
 import java.math.BigInteger;
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
@@ -209,7 +210,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.startsWith;
-import static org.junit.Assert.assertThat;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -1478,10 +1478,24 @@ public Path getDataPath(String relativePath) {
         // we override LTC behavior here: wrap even resources with mockfilesystems,
         // because some code is buggy when it comes to multiple nio.2 filesystems
         // (e.g. FileSystemUtils, and likely some tests)
+        return getResourceDataPath(getClass(), relativePath);
+    }
+
+    public static Path getResourceDataPath(Class<?> clazz, String relativePath) {
+        final var resource = Objects.requireNonNullElseGet(
+            clazz.getResource(relativePath),
+            () -> fail(null, "resource not found: [%s][%s]", clazz.getCanonicalName(), relativePath)
+        );
+        final URI uri;
+        try {
+            uri = resource.toURI();
+        } catch (Exception e) {
+            return fail(null, "resource URI not found: [%s][%s]", clazz.getCanonicalName(), relativePath);
+        }
         try {
-            return PathUtils.get(getClass().getResource(relativePath).toURI()).toAbsolutePath().normalize();
+            return PathUtils.get(uri).toAbsolutePath().normalize();
         } catch (Exception e) {
-            throw new RuntimeException("resource not found: " + relativePath, e);
+            return fail(e, "resource path not found: %s", uri);
         }
     }
 

test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ssl.KeyStoreUtil;
+import org.junit.rules.ExternalResource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir;
+
+public class TestTrustStore extends ExternalResource {
+
+    private final CheckedSupplier<InputStream, IOException> pemStreamSupplier;
+
+    public TestTrustStore(CheckedSupplier<InputStream, IOException> pemStreamSupplier) {
+        this.pemStreamSupplier = pemStreamSupplier;
+    }
+
+    private Path trustStorePath;
+
+    public Path getTrustStorePath() {
+        return Objects.requireNonNullElseGet(trustStorePath, () -> ESTestCase.fail(null, "trust store not created"));
+    }
+
+    @Override
+    protected void before() {
+        final var tmpDir = createTempDir();
+        final var tmpTrustStorePath = tmpDir.resolve("trust-store.jks");
+        try (var pemStream = pemStreamSupplier.get(); var jksStream = Files.newOutputStream(tmpTrustStorePath)) {
+            final List<Certificate> certificates = CertificateFactory.getInstance("X.509")
+                .generateCertificates(pemStream)
+                .stream()
+                .map(i -> (Certificate) i)
+                .toList();
+            final var trustStore = KeyStoreUtil.buildTrustStore(certificates);
+            trustStore.store(jksStream, null);
+            trustStorePath = tmpTrustStorePath;
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+}

x-pack/plugin/repositories-metering-api/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase;
 import org.junit.ClassRule;
@@ -26,7 +27,15 @@ public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPI
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER
+    );
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
@@ -44,13 +53,14 @@ public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPI
         )
         .setting(
             "azure.client.repositories_metering.endpoint_suffix",
-            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
+            () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(),
             s -> USE_FIXTURE
         )
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_FIXTURE)
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     @Override
     protected String repositoryType() {