diff --git a/plugin/trino-kudu/pom.xml b/plugin/trino-kudu/pom.xml index 9c74c58348e9..694c20e7e87a 100644 --- a/plugin/trino-kudu/pom.xml +++ b/plugin/trino-kudu/pom.xml @@ -176,6 +176,13 @@ test + + org.testcontainers + toxiproxy + test + + + org.testng testng diff --git a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestingKuduServer.java b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestingKuduServer.java index 13605d8baafc..893deb3a677b 100644 --- a/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestingKuduServer.java +++ b/plugin/trino-kudu/src/test/java/io/trino/plugin/kudu/TestingKuduServer.java @@ -18,45 +18,77 @@ import com.google.common.net.HostAndPort; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; +import org.testcontainers.containers.ToxiproxyContainer; import java.io.Closeable; import java.io.IOException; +import java.net.Inet4Address; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Enumeration; import java.util.List; +import static java.lang.String.format; + public class TestingKuduServer implements Closeable { + private static final String KUDU_IMAGE = "apache/kudu:1.10.0"; private static final Integer KUDU_MASTER_PORT = 7051; private static final Integer KUDU_TSERVER_PORT = 7050; private static final Integer NUMBER_OF_REPLICA = 3; + private static final String TOXIPROXY_IMAGE = "shopify/toxiproxy:2.1.0"; + private static final String TOXIPROXY_NETWORK_ALIAS = "toxiproxy"; + + private final Network network; + private final ToxiproxyContainer toxiProxy; private final GenericContainer master; private final List> tServers; + /** + * Kudu tablets needs to know the host/mapped port it will be bound to in order to configure --rpc_advertised_addresses + * However when using non-fixed ports in testcontainers, we only know the mapped port after the container starts up + * In order to workaround this, create a proxy to forward traffic from the host to the underlying tablets + * Since the ToxiProxy container starts up *before* kudu, we know the mapped port when configuring the kudu tablets + */ public TestingKuduServer() { - Network network = Network.newNetwork(); + network = Network.newNetwork(); ImmutableList.Builder> tServersBuilder = ImmutableList.builder(); - this.master = new GenericContainer<>("apache/kudu:1.10.0") + + String hostIP = getHostIPAddress(); + + String masterContainerAlias = "kudu-master"; + this.master = new GenericContainer<>(KUDU_IMAGE) .withExposedPorts(KUDU_MASTER_PORT) .withCommand("master") .withNetwork(network) - .withNetworkAliases("kudu-master"); + .withNetworkAliases(masterContainerAlias); + + toxiProxy = new ToxiproxyContainer(TOXIPROXY_IMAGE) + .withNetwork(network) + .withNetworkAliases(TOXIPROXY_NETWORK_ALIAS); + toxiProxy.start(); for (int instance = 0; instance < NUMBER_OF_REPLICA; instance++) { String instanceName = "kudu-tserver-" + instance; - GenericContainer tableServer = new GenericContainer<>("apache/kudu:1.10.0") + ToxiproxyContainer.ContainerProxy proxy = toxiProxy.getProxy(instanceName, KUDU_TSERVER_PORT); + GenericContainer tableServer = new GenericContainer<>(KUDU_IMAGE) .withExposedPorts(KUDU_TSERVER_PORT) .withCommand("tserver") - .withEnv("KUDU_MASTERS", "kudu-master:" + KUDU_MASTER_PORT) + .withEnv("KUDU_MASTERS", format("%s:%s", masterContainerAlias, KUDU_MASTER_PORT)) + .withEnv("TSERVER_ARGS", format("--fs_wal_dir=/var/lib/kudu/tserver --logtostderr --use_hybrid_clock=false --rpc_bind_addresses=%s:%s --rpc_advertised_addresses=%s:%s", instanceName, KUDU_TSERVER_PORT, hostIP, proxy.getProxyPort())) .withNetwork(network) - .withNetworkAliases("kudu-tserver-" + instance) - .dependsOn(master) - .withEnv("TSERVER_ARGS", "--fs_wal_dir=/var/lib/kudu/tserver --use_hybrid_clock=false --rpc_advertised_addresses=" + instanceName); + .withNetworkAliases(instanceName) + .dependsOn(master); + tServersBuilder.add(tableServer); } this.tServers = tServersBuilder.build(); master.start(); + tServers.forEach(GenericContainer::start); } @@ -71,9 +103,32 @@ public void close() try (Closer closer = Closer.create()) { closer.register(master::stop); tServers.forEach(tabletServer -> closer.register(tabletServer::stop)); + closer.register(toxiProxy::stop); + closer.register(network::close); } catch (IOException e) { throw new RuntimeException(e); } } + + private static String getHostIPAddress() + { + // Binding kudu's `rpc_advertised_addresses` to 127.0.0.1 inside the container will not bind to the host's loopback address + // As a workaround, use a site local ipv4 address from the host + // This is roughly equivalent to setting the KUDU_QUICKSTART_IP defined here: https://kudu.apache.org/docs/quickstart.html#_set_kudu_quickstart_ip + try { + Enumeration networkInterfaceEnumeration = NetworkInterface.getNetworkInterfaces(); + while (networkInterfaceEnumeration.hasMoreElements()) { + for (InterfaceAddress interfaceAddress : networkInterfaceEnumeration.nextElement().getInterfaceAddresses()) { + if (interfaceAddress.getAddress().isSiteLocalAddress() && interfaceAddress.getAddress() instanceof Inet4Address) { + return interfaceAddress.getAddress().getHostAddress(); + } + } + } + } + catch (SocketException e) { + throw new RuntimeException(e); + } + throw new IllegalStateException("Could not find site local ipv4 address, failed to launch kudu"); + } }