Debug multinode worker

presto-main/src/test/java/io/prestosql/tests/LogTestDurationListener.java

@@ -48,6 +48,10 @@ public class LogTestDurationListener
 {
     private static final Logger LOG = Logger.get(LogTestDurationListener.class);
 
+    // LogTestDurationListener does not support concurrent invocations of same test method
+    // (@Test(threadPoolSize=n)), so we need kill switch for local development purposes.
+    private static final boolean enabled = Boolean.getBoolean("LogTestDurationListener.enabled");
+
     private static final Duration SINGLE_TEST_LOGGING_THRESHOLD = Duration.valueOf("30s");
     private static final Duration CLASS_LOGGING_THRESHOLD = Duration.valueOf("1m");
     // Must be below Travis "no output" timeout (10m). E.g. TestElasticsearchIntegrationSmokeTest is known to take ~5-6m.
@@ -70,6 +74,10 @@ public LogTestDurationListener()
     @Override
     public synchronized void onExecutionStart()
     {
+        if (!enabled) {
+            return;
+        }
+
         resetHangMonitor();
         finished.set(false);
         if (monitorHangTask == null) {
@@ -80,6 +88,10 @@ public synchronized void onExecutionStart()
     @Override
     public synchronized void onExecutionFinish()
     {
+        if (!enabled) {
+            return;
+        }
+
         resetHangMonitor();
         finished.set(true);
         // do not stop hang task so notification of hung test JVM will fire
@@ -133,12 +145,20 @@ private void resetHangMonitor()
     @Override
     public void onBeforeClass(ITestClass testClass)
     {
+        if (!enabled) {
+            return;
+        }
+
         beginTest(getName(testClass));
     }
 
     @Override
     public void onAfterClass(ITestClass testClass)
     {
+        if (!enabled) {
+            return;
+        }
+
         String name = getName(testClass);
         Duration duration = endTest(name);
         if (duration.compareTo(CLASS_LOGGING_THRESHOLD) > 0) {
@@ -149,12 +169,20 @@ public void onAfterClass(ITestClass testClass)
     @Override
     public void beforeInvocation(IInvokedMethod method, ITestResult testResult)
     {
+        if (!enabled) {
+            return;
+        }
+
         beginTest(getName(method));
     }
 
     @Override
     public void afterInvocation(IInvokedMethod method, ITestResult testResult)
     {
+        if (!enabled) {
+            return;
+        }
+
         String name = getName(method);
         Duration duration = endTest(name);
         if (duration.compareTo(SINGLE_TEST_LOGGING_THRESHOLD) > 0) {

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/EnvironmentUp.java

@@ -21,12 +21,12 @@
 import io.airlift.log.Logger;
 import io.prestosql.tests.product.launcher.Extensions;
 import io.prestosql.tests.product.launcher.LauncherModule;
+import io.prestosql.tests.product.launcher.docker.ContainerUtil;
 import io.prestosql.tests.product.launcher.env.Environment;
 import io.prestosql.tests.product.launcher.env.EnvironmentFactory;
 import io.prestosql.tests.product.launcher.env.EnvironmentModule;
 import io.prestosql.tests.product.launcher.env.EnvironmentOptions;
 import io.prestosql.tests.product.launcher.env.Environments;
-import io.prestosql.tests.product.launcher.testcontainers.TestcontainersUtil;
 import org.testcontainers.DockerClientFactory;
 
 import javax.inject.Inject;
@@ -132,7 +132,7 @@ private void killContainersReaperContainer()
         {
             try (DockerClient dockerClient = DockerClientFactory.lazyClient()) {
                 log.info("Killing the testcontainers reaper container (Ryuk) so that environment can stay alive");
-                TestcontainersUtil.killContainersReaperContainer(dockerClient);
+                ContainerUtil.killContainersReaperContainer(dockerClient);
             }
             catch (IOException e) {
                 throw new UncheckedIOException(e);

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java

@@ -43,8 +43,8 @@
 
 import static com.google.common.base.Preconditions.checkState;
 import static io.prestosql.tests.product.launcher.cli.Commands.runCommand;
+import static io.prestosql.tests.product.launcher.docker.ContainerUtil.exposePort;
 import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_TEMPTO_PROFILE_CONFIG;
-import static io.prestosql.tests.product.launcher.testcontainers.TestcontainersUtil.exposePort;
 import static java.util.Objects.requireNonNull;
 import static org.testcontainers.containers.BindMode.READ_ONLY;
 

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/docker/ContainerUtil.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.prestosql.tests.product.launcher.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerCmd;
+import com.github.dockerjava.api.command.ListContainersCmd;
+import com.github.dockerjava.api.command.ListNetworksCmd;
+import com.github.dockerjava.api.exception.ConflictException;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.AccessMode;
+import com.github.dockerjava.api.model.Bind;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.api.model.Network;
+import io.prestosql.tests.product.launcher.env.DockerContainer;
+import org.testcontainers.DockerClientFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkState;
+import static io.prestosql.tests.product.launcher.docker.DockerFiles.createTemporaryDirectoryForDocker;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.isRegularFile;
+import static java.nio.file.Files.readAllLines;
+import static java.nio.file.Files.write;
+import static java.util.Objects.requireNonNull;
+
+public final class ContainerUtil
+{
+    private ContainerUtil() {}
+
+    public static void killContainers(DockerClient dockerClient, Function<ListContainersCmd, ListContainersCmd> filter)
+    {
+        while (true) {
+            ListContainersCmd listContainersCmd = filter.apply(dockerClient.listContainersCmd()
+                    .withShowAll(true));
+
+            List<Container> containers = listContainersCmd.exec();
+            if (containers.isEmpty()) {
+                break;
+            }
+            for (Container container : containers) {
+                try {
+                    dockerClient.removeContainerCmd(container.getId())
+                            .withForce(true)
+                            .exec();
+                }
+                catch (ConflictException | NotFoundException ignored) {
+                }
+            }
+        }
+    }
+
+    public static void removeNetworks(DockerClient dockerClient, Function<ListNetworksCmd, ListNetworksCmd> filter)
+    {
+        ListNetworksCmd listNetworksCmd = filter.apply(dockerClient.listNetworksCmd());
+        List<Network> networks = listNetworksCmd.exec();
+        for (Network network : networks) {
+            dockerClient.removeNetworkCmd(network.getId())
+                    .exec();
+        }
+    }
+
+    public static void killContainersReaperContainer(DockerClient dockerClient)
+    {
+        @SuppressWarnings("resource")
+        Void ignore = dockerClient.removeContainerCmd("testcontainers-ryuk-" + DockerClientFactory.SESSION_ID)
+                .withForce(true)
+                .exec();
+    }
+
+    public static void exposePort(DockerContainer container, int port)
+    {
+        container.addExposedPort(port);
+        container.withFixedExposedPort(port, port);
+    }
+
+    public static void enableJavaDebugger(DockerContainer container, String jvmConfigPath, int debugPort)
+    {
+        requireNonNull(jvmConfigPath, "jvmConfigPath is null");
+        container.withCreateContainerCmdModifier(createContainerCmd -> enableDebuggerInJvmConfig(createContainerCmd, jvmConfigPath, debugPort));
+        exposePort(container, debugPort);
+    }
+
+    private static void enableDebuggerInJvmConfig(CreateContainerCmd createContainerCmd, String jvmConfigPath, int debugPort)
+    {
+        try {
+            Bind[] binds = firstNonNull(createContainerCmd.getBinds(), new Bind[0]);
+            boolean found = false;
+
+            // Last bind wins, so we can find the last one only
+            for (int bindIndex = binds.length - 1; bindIndex >= 0; bindIndex--) {
+                Bind bind = binds[bindIndex];
+                if (!bind.getVolume().getPath().equals(jvmConfigPath)) {
+                    continue;
+                }
+
+                Path hostJvmConfig = Paths.get(bind.getPath());
+                checkState(isRegularFile(hostJvmConfig), "Bind for %s is not a file", jvmConfigPath);
+
+                Path temporaryDirectory = createTemporaryDirectoryForDocker();
+                Path newJvmConfig = temporaryDirectory.resolve("jvm.config");
+
+                List<String> jvmOptions = new ArrayList<>(readAllLines(hostJvmConfig, UTF_8));
+                jvmOptions.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:" + debugPort);
+                write(newJvmConfig, jvmOptions, UTF_8);
+
+                binds[bindIndex] = new Bind(newJvmConfig.toString(), bind.getVolume(), AccessMode.ro, bind.getSecMode(), bind.getNoCopy(), bind.getPropagationMode());
+                found = true;
+                break;
+            }
+
+            checkState(found, "Could not find %s bind", jvmConfigPath);
+            createContainerCmd.withBinds(binds);
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environments.java

@@ -17,7 +17,6 @@
 import com.google.common.base.CaseFormat;
 import com.google.common.reflect.ClassPath;
 import io.airlift.log.Logger;
-import io.prestosql.tests.product.launcher.docker.DockerUtil;
 import io.prestosql.tests.product.launcher.env.common.TestsEnvironment;
 import org.testcontainers.DockerClientFactory;
 import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
@@ -27,6 +26,8 @@
 import java.util.List;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static io.prestosql.tests.product.launcher.docker.ContainerUtil.killContainers;
+import static io.prestosql.tests.product.launcher.docker.ContainerUtil.removeNetworks;
 import static io.prestosql.tests.product.launcher.env.Environment.PRODUCT_TEST_LAUNCHER_NETWORK;
 import static io.prestosql.tests.product.launcher.env.Environment.PRODUCT_TEST_LAUNCHER_STARTED_LABEL_NAME;
 import static io.prestosql.tests.product.launcher.env.Environment.PRODUCT_TEST_LAUNCHER_STARTED_LABEL_VALUE;
@@ -41,10 +42,10 @@ public static void pruneEnvironment()
     {
         log.info("Shutting down previous containers");
         try (DockerClient dockerClient = DockerClientFactory.lazyClient()) {
-            DockerUtil.killContainers(
+            killContainers(
                     dockerClient,
                     listContainersCmd -> listContainersCmd.withLabelFilter(ImmutableMap.of(PRODUCT_TEST_LAUNCHER_STARTED_LABEL_NAME, PRODUCT_TEST_LAUNCHER_STARTED_LABEL_VALUE)));
-            DockerUtil.removeNetworks(
+            removeNetworks(
                     dockerClient,
                     listNetworksCmd -> listNetworksCmd.withNameFilter(PRODUCT_TEST_LAUNCHER_NETWORK));
         }