[ci] Add PR pipeline for testing ipv6

.buildkite/hooks/pre-command

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+if [[ "${USE_IPV6_TESTING:-}" == "true" ]]; then
+  # Ensure that `localhost` resolves to only `::1`
+  sudo bash -c 'echo "::1   localhost" | cat - /etc/hosts > temp_file && mv temp_file /etc/hosts'
+  sudo sed -i '/^127\.0\.0\.1/d' /etc/hosts
+  sudo resolvectl flush-caches || true
+fi
+
 # On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests
 # This fixes that
 chmod 755 ~

.buildkite/pipelines/periodic.weekly.yml

@@ -0,0 +1,30 @@
+steps:
+  - group: ipv6-tests
+    steps:
+      - label: "{{matrix.GRADLE_TASK}} / ipv6-tests"
+        command: .ci/scripts/run-gradle.sh --continue -Djava.net.preferIPv6Addresses=true -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
+        timeout_in_minutes: 300
+        agents:
+          provider: aws
+          imagePrefix: elasticsearch-aws-ubuntu-2404-nvidia
+          instanceType: m7a.8xlarge
+          diskSizeGb: 150
+          diskType: gp3
+          diskName: /dev/sda1
+          buildDirectory: /dev/shm/bk
+          ipv6Only: true
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+          JAVA_TOOL_OPTIONS: "-Djava.net.preferIPv6Addresses=true"
+          _JAVA_OPTIONS: "-Djava.net.preferIPv6Addresses=true"
+          USE_IPV6_TESTING: "true"
+          DISABLE_DOCKER_TESTS: "true"
+        matrix:
+          setup:
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6

.buildkite/pipelines/pull-request/ipv6.yml

@@ -0,0 +1,35 @@
+config:
+  allow-labels: test-ipv6
+  skip-labels:
+    - ">test-mute"
+steps:
+  - group: ipv6-tests
+    steps:
+      - label: "{{matrix.GRADLE_TASK}} / ipv6-tests"
+        key: "ipv6-tests"
+        command: .ci/scripts/run-gradle.sh --continue -Djava.net.preferIPv6Addresses=true -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
+        timeout_in_minutes: 300
+        agents:
+          provider: aws
+          imagePrefix: elasticsearch-aws-ubuntu-2404-nvidia
+          instanceType: m7a.8xlarge
+          diskSizeGb: 150
+          diskType: gp3
+          diskName: /dev/sda1
+          buildDirectory: /dev/shm/bk
+          ipv6Only: true
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+          JAVA_TOOL_OPTIONS: "-Djava.net.preferIPv6Addresses=true"
+          _JAVA_OPTIONS: "-Djava.net.preferIPv6Addresses=true"
+          USE_IPV6_TESTING: "true"
+          DISABLE_DOCKER_TESTS: "true"
+        matrix:
+          setup:
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1370,6 +1370,9 @@ private void createConfiguration() {
         baseConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
         baseConfig.put("path.data", confPathData.toAbsolutePath().toString());
         baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());
+        if (Boolean.getBoolean("java.net.preferIPv6Addresses")) {
+            baseConfig.put("network.host", "_local:ipv6_");
+        }
         baseConfig.put("node.attr.testattr", "test");
         baseConfig.put("node.portsfile", "true");
         baseConfig.put("http.port", httpPort);

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/HttpClientTests.java

@@ -14,6 +14,7 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -117,7 +118,7 @@ public static void stopServer() {
     }
 
     private static String url(final String path) {
-        String hostname = server.getAddress().getHostString();
+        String hostname = InetAddresses.toUriString(server.getAddress().getAddress());
         int port = server.getAddress().getPort();
         return "http://" + hostname + ":" + port + path;
     }

modules/reindex/build.gradle

@@ -31,7 +31,7 @@ testClusters.configureEach {
   module ':modules:lang-painless'
   module ':modules:rest-root'
   // Whitelist reindexing from the local node so we can test reindex-from-remote.
-  setting 'reindex.remote.whitelist', '127.0.0.1:*'
+  setting 'reindex.remote.whitelist', '127.0.0.1:*,[::1]:*'
 }
 
 dependencies {

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java

@@ -74,7 +74,7 @@ protected boolean addMockHttpTransport() {
     protected Settings nodeSettings() {
         Settings.Builder settings = Settings.builder().put(super.nodeSettings());
         // Whitelist reindexing from the http host we're going to use
-        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*");
+        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*,::1:*");
         settings.put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME);
         return settings.build();
     }

modules/reindex/src/test/java/org/elasticsearch/reindex/RetryTests.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.BackoffPolicy;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -91,7 +92,7 @@ protected boolean addMockHttpTransport() {
     final Settings nodeSettings() {
         return Settings.builder()
             // whitelist reindexing from the HTTP host we're going to use
-            .put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*")
+            .put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "127.0.0.1:*,[::1]:*")
             .build();
     }
 
@@ -118,9 +119,10 @@ public void testReindexFromRemote() throws Exception {
             assertNotNull(masterNode);
 
             TransportAddress address = masterNode.getInfo(HttpInfo.class).getAddress().publishAddress();
+            String host = InetAddresses.toUriString(address.address().getAddress());
             RemoteInfo remote = new RemoteInfo(
                 "http",
-                address.getAddress(),
+                host,
                 address.getPort(),
                 null,
                 new BytesArray("{\"match_all\":{}}"),

modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java

@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() throws Exception {
         .module("lang-painless")
         .module("parent-join")
         .module("rest-root")
-        .setting("reindex.remote.whitelist", "127.0.0.1:*")
+        .setting("reindex.remote.whitelist", "127.0.0.1:*,[::1]:*")
         .build();
 
     @Override

modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -146,7 +146,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         }
 
         // see com.azure.storage.blob.BlobUrlParts.parseIpUrl
-        final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrl() + "/" + accountName;
+        final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrlLocalhost() + "/" + accountName;
 
         // The first node configured sets these for all nodes
         MAX_CONNECTION_SETTING.compareAndSet(-1, randomIntBetween(10, 30));

modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java

@@ -218,7 +218,9 @@ protected static Optional<Integer> getRangeEnd(HttpExchange exchange) {
 
     protected String getEndpointForServer(HttpServer server, String accountName) {
         InetSocketAddress address = server.getAddress();
-        return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort() + "/" + accountName;
+        // Use "localhost" for loopback addresses to work around Azure SDK's inability to parse bracketed IPv6 addresses
+        String host = address.getAddress().isLoopbackAddress() ? "localhost" : InetAddresses.toUriString(address.getAddress());
+        return "http://" + host + ":" + address.getPort() + "/" + accountName;
     }
 
     public static void readFromInputStream(InputStream inputStream, long bytesToRead) {

modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java

@@ -797,7 +797,9 @@ private void warnIfUnsupportedSettingSet(String settingName, Object value) {
 
     private String getEndpointForServer(HttpServer server, String accountName) {
         InetSocketAddress address = server.getAddress();
-        return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort() + "/" + accountName;
+        // Use "localhost" for loopback addresses to work around Azure SDK's inability to parse bracketed IPv6 addresses
+        String host = address.getAddress().isLoopbackAddress() ? "localhost" : InetAddresses.toUriString(address.getAddress());
+        return "http://" + host + ":" + address.getPort() + "/" + accountName;
     }
 
     @Override

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GcsProxyIntegrationTests.java

@@ -14,6 +14,7 @@
 
 import com.sun.net.httpserver.HttpServer;
 
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
@@ -97,7 +98,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace("test").getKey(),
             TestUtils.createServiceAccount(random())
         );
-        String upstreamServerUrl = "http://" + upstreamServer.getAddress().getHostString() + ":" + upstreamServer.getAddress().getPort();
+        String host = InetAddresses.toUriString(upstreamServer.getAddress().getAddress());
+        String upstreamServerUrl = "http://" + host + ":" + upstreamServer.getAddress().getPort();
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
             .put(ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), upstreamServerUrl)

modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerStatsTests.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.common.blobstore.support.BlobMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -279,7 +280,8 @@ private ContainerAndBlobStore createBlobContainer(final String repositoryName) t
 
     protected String getEndpointForServer(final HttpServer server) {
         final InetSocketAddress address = server.getAddress();
-        return "http://" + address.getHostString() + ":" + address.getPort();
+        String host = InetAddresses.toUriString(address.getAddress());
+        return "http://" + host + ":" + address.getPort();
     }
 
     private record ContainerAndBlobStore(GoogleCloudStorageBlobContainer blobContainer, GoogleCloudStorageBlobStore blobStore)

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ExplicitProtocolRestIT.java

@@ -48,7 +48,7 @@ public class RepositoryS3ExplicitProtocolRestIT extends AbstractRepositoryS3Rest
     private static String getEndpoint() {
         final var s3FixtureAddress = s3Fixture.getAddress();
         assertThat(s3FixtureAddress, startsWith("http://"));
-        return s3FixtureAddress.substring("http://".length());
+        return "\"" + s3FixtureAddress.substring("http://".length()) + "\"";
     }
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java

@@ -139,7 +139,7 @@ public void setUp() throws Exception {
             DEFAULT_REGION_UNAVAILABLE
         ) {
             private InetAddress[] resolveHost(String host) throws UnknownHostException {
-                assertEquals("127.0.0.1", host);
+                assertTrue(InetAddress.getByName(host).isLoopbackAddress());
                 if (shouldErrorOnDns && randomBoolean() && randomBoolean()) {
                     throw new UnknownHostException(host);
                 }
@@ -204,7 +204,8 @@ protected BlobContainer createBlobContainer(
         final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
 
         final InetSocketAddress address = httpServer.getAddress();
-        final String endpoint = "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort();
+        String host = InetAddresses.toUriString(address.getAddress());
+        final String endpoint = "http://" + host + ":" + address.getPort();
         logger.info("--> creating client with endpoint [{}]", endpoint);
         clientSettings.put(ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint);
 

modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java

@@ -64,11 +64,14 @@ public void testDifferentPorts() throws Exception {
                 assertThat(boundTransportAddress.boundAddresses()[0].getPort(), equalTo(boundTransportAddress.publishAddress().getPort()));
             } else {
                 assertThat(boundTransportAddress.boundAddresses().length, greaterThan(1));
+                // Determine if the publish address is IPv4 or IPv6
+                boolean publishIsV4 = boundTransportAddress.publishAddress().address().getAddress() instanceof Inet4Address;
                 for (TransportAddress boundAddress : boundTransportAddress.boundAddresses()) {
                     assertThat(boundAddress, instanceOf(TransportAddress.class));
                     TransportAddress inetBoundAddress = boundAddress;
-                    if (inetBoundAddress.address().getAddress() instanceof Inet4Address) {
-                        // IPv4 address is preferred publish address for _local_
+                    boolean boundIsV4 = inetBoundAddress.address().getAddress() instanceof Inet4Address;
+                    if (boundIsV4 == publishIsV4) {
+                        // The bound address matching the publish address type should have the same port
                         assertThat(inetBoundAddress.getPort(), equalTo(boundTransportAddress.publishAddress().getPort()));
                     }
                 }

modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -92,7 +93,9 @@ public void testConnectException() throws UnknownHostException {
             null
         );
         assertThat(e.getMessage(), containsString("connect_exception"));
-        assertThat(e.getMessage(), containsString("[127.0.0.1:9876]"));
+
+        String loopback = InetAddresses.toUriString(InetAddress.getByName("localhost"));
+        assertThat(e.getMessage(), containsString("[" + loopback + ":9876]"));
     }
 
     public void testDefaultKeepAliveSettings() throws IOException {

plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
@@ -88,6 +89,17 @@ protected void before() {
         }
     };
 
+    @ClassRule
+    public static final ExternalResource MUTE_IN_IPV6 = new ExternalResource() {
+        @Override
+        protected void before() {
+            assumeFalse(
+                "Tests not currently working correctly with IPv6",
+                Booleans.parseBoolean(System.getProperty("java.net.preferIPv6Addresses", "true"))
+            );
+        }
+    };
+
     @BeforeClass
     public static void setupKeyStore() throws IOException {
         Path tempDir = createTempDir();

plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2NetworkAddressesTestCase.java

@@ -33,10 +33,13 @@ protected boolean addMockHttpTransport() {
 
     void verifyPublishAddress(String publishAddressSetting, String expectedAddress) throws IOException {
         final var node = internalCluster().startNode(Settings.builder().put("http.publish_host", publishAddressSetting));
-        assertEquals(
-            expectedAddress,
-            internalCluster().getInstance(HttpServerTransport.class, node).boundAddress().publishAddress().getAddress()
-        );
+        String actualAddress = internalCluster().getInstance(HttpServerTransport.class, node).boundAddress().publishAddress().getAddress();
+
+        if (actualAddress.equals("::1") && expectedAddress.equals("127.0.0.1")) {
+            expectedAddress = "::1";
+        }
+
+        assertEquals(expectedAddress, actualAddress);
         internalCluster().stopNode(node);
     }
 }

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java

@@ -17,6 +17,7 @@
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.PageCacheRecycler;
@@ -30,12 +31,12 @@
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.hamcrest.Matchers.aMapWithSize;
@@ -63,7 +64,11 @@ protected MockTransportService createTransportService() {
 
     public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
         final String accessKey = "ec2_access";
-        final List<String> hosts = List.of("127.0.0.1:9300");
+        final InetAddress loopbackAddress = InetAddress.getLoopbackAddress();
+        // For the EC2 response, we use the plain IP address without port
+        final String loopbackAddressString = NetworkAddress.format(loopbackAddress);
+        // For the expected TransportAddress, use proper IPv6 formatting with brackets
+        final String expectedHost = NetworkAddress.format(loopbackAddress, 9300);
         final Map<String, Integer> failedRequests = new ConcurrentHashMap<>();
         // retry the same request 5 times at most
         final int maxRetries = randomIntBetween(1, 5);
@@ -88,9 +93,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
                     for (NameValuePair parse : URLEncodedUtils.parse(request, UTF_8)) {
                         if ("Action".equals(parse.getName())) {
                             responseBody = generateDescribeInstancesResponse(
-                                hosts.stream()
-                                    .map(address -> Instance.builder().publicIpAddress(address).build())
-                                    .collect(Collectors.toList())
+                                List.of(Instance.builder().publicIpAddress(loopbackAddressString).build())
                             );
                             break;
                         }
@@ -111,7 +114,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
             resolver.start();
             final List<TransportAddress> addressList = seedHostsProvider.getSeedAddresses(resolver);
             assertThat(addressList, Matchers.hasSize(1));
-            assertThat(addressList.get(0).toString(), is(hosts.get(0)));
+            assertThat(addressList.get(0).toString(), is(expectedHost));
             assertThat(failedRequests, aMapWithSize(1));
             assertThat(failedRequests.values().iterator().next(), is(maxRetries));
         }