From 40d51fe305207297f3391a35a3ca76329adbdca8 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Mon, 15 Jan 2018 21:45:18 +0100 Subject: [PATCH 1/8] Start a helper "Ryuk" container to make sure that we clean the containers after the execution even if the JVM was "kill -9"ed --- .../testcontainers/DockerClientFactory.java | 81 +++++------- .../containers/DockerComposeContainer.java | 48 ++----- .../containers/GenericContainer.java | 9 +- .../testcontainers/containers/Network.java | 11 +- .../utility/ResourceReaper.java | 123 ++++++++++++++++++ .../utility/TestcontainersConfiguration.java | 4 + 6 files changed, 183 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/org/testcontainers/DockerClientFactory.java b/core/src/main/java/org/testcontainers/DockerClientFactory.java index c7568fd8b40..efe322036d9 100644 --- a/core/src/main/java/org/testcontainers/DockerClientFactory.java +++ b/core/src/main/java/org/testcontainers/DockerClientFactory.java @@ -2,35 +2,32 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.exception.InternalServerErrorException; import com.github.dockerjava.api.exception.NotFoundException; import com.github.dockerjava.api.model.*; import com.github.dockerjava.core.command.ExecStartResultCallback; import com.github.dockerjava.core.command.PullImageResultCallback; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.rnorth.ducttape.unreliables.Unreliables; import org.rnorth.visibleassertions.VisibleAssertions; -import org.testcontainers.dockerclient.*; +import org.testcontainers.dockerclient.DockerClientProviderStrategy; +import org.testcontainers.dockerclient.DockerMachineClientProviderStrategy; import org.testcontainers.utility.ComparableVersion; -import org.testcontainers.utility.MountableFile; +import org.testcontainers.utility.ResourceReaper; import org.testcontainers.utility.TestcontainersConfiguration; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStream; -import java.net.Socket; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.ServiceLoader; -import java.util.concurrent.TimeUnit; +import java.util.UUID; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -42,12 +39,22 @@ @Slf4j public class DockerClientFactory { + public static final String TESTCONTAINERS_LABEL = DockerClientFactory.class.getPackage().getName(); + public static final String TESTCONTAINERS_SESSION_ID_LABEL = TESTCONTAINERS_LABEL + ".sessionId"; + + public static final String SESSION_ID = UUID.randomUUID().toString(); + + public static final Map DEFAULT_LABELS = ImmutableMap.of( + TESTCONTAINERS_LABEL, "true", + TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID + ); + private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyImage(); private static DockerClientFactory instance; // Cached client configuration private DockerClientProviderStrategy strategy; - private boolean preconditionsChecked = false; + private boolean initialized = false; private String activeApiVersion; private String activeExecutionDriver; @@ -95,7 +102,7 @@ public DockerClient client() { log.info("Docker host IP address is {}", hostIpAddress); DockerClient client = strategy.getClient(); - if (!preconditionsChecked) { + if (!initialized) { Info dockerInfo = client.infoCmd().exec(); Version version = client.versionCmd().exec(); activeApiVersion = version.getApiVersion(); @@ -106,30 +113,19 @@ public DockerClient client() { " Operating System: " + dockerInfo.getOperatingSystem() + "\n" + " Total Memory: " + dockerInfo.getMemTotal() / (1024 * 1024) + " MB"); - if (!TestcontainersConfiguration.getInstance().isDisableChecks()) { - VisibleAssertions.info("Checking the system..."); - - checkDockerVersion(version.getVersion()); - - MountableFile mountableFile = MountableFile.forClasspathResource(this.getClass().getName().replace(".", "/") + ".class"); + String ryukContainerId = ResourceReaper.start(hostIpAddress, client); + log.info("Ryuk started"); - runInsideDocker( - client, - cmd -> cmd - .withCmd("/bin/sh", "-c", "while true; do printf 'hello' | nc -l -p 80; done") - .withBinds(new Bind(mountableFile.getResolvedPath(), new Volume("/dummy"), AccessMode.ro)) - .withExposedPorts(new ExposedPort(80)) - .withPublishAllPorts(true), - (dockerClient, id) -> { + VisibleAssertions.info("Checking the system..."); - checkDiskSpace(dockerClient, id); - checkMountableFile(dockerClient, id); - checkExposedPort(hostIpAddress, dockerClient, id); + checkDockerVersion(version.getVersion()); - return null; - }); + if (!TestcontainersConfiguration.getInstance().isDisableChecks()) { + checkDiskSpace(client, ryukContainerId); + checkMountableFile(client, ryukContainerId); } - preconditionsChecked = true; + + initialized = true; } return client; @@ -178,22 +174,6 @@ private void checkMountableFile(DockerClient dockerClient, String id) { } } - private void checkExposedPort(String hostIpAddress, DockerClient dockerClient, String id) { - String response = Unreliables.retryUntilSuccess(3, TimeUnit.SECONDS, () -> { - InspectContainerResponse inspectedContainer = dockerClient.inspectContainerCmd(id).exec(); - - String portSpec = inspectedContainer.getNetworkSettings().getPorts().getBindings().values().iterator().next()[0].getHostPortSpec(); - - try (Socket socket = new Socket(hostIpAddress, Integer.parseInt(portSpec))) { - return IOUtils.toString(socket.getInputStream(), Charset.defaultCharset()); - } catch (IOException e) { - return e.getMessage(); - } - }); - - VisibleAssertions.assertEquals("A port exposed by a docker container should be accessible", "hello", response); - } - /** * Check whether the image is available locally and pull it otherwise */ @@ -221,7 +201,8 @@ public T runInsideDocker(Consumer createContainerCmdCons private T runInsideDocker(DockerClient client, Consumer createContainerCmdConsumer, BiFunction block) { checkAndPullImage(client, TINY_IMAGE); - CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE); + CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE) + .withLabels(DEFAULT_LABELS); createContainerCmdConsumer.accept(createContainerCmd); String id = createContainerCmd.exec().getId(); @@ -263,7 +244,7 @@ DiskSpaceUsage parseAvailableDiskSpace(String dfOutput) { * @return the docker API version of the daemon that we have connected to */ public String getActiveApiVersion() { - if (!preconditionsChecked) { + if (!initialized) { client(); } return activeApiVersion; @@ -273,7 +254,7 @@ public String getActiveApiVersion() { * @return the docker execution driver of the daemon that we have connected to */ public String getActiveExecutionDriver() { - if (!preconditionsChecked) { + if (!initialized) { client(); } return activeExecutionDriver; diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index b718351c86b..821f9382707 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -1,7 +1,6 @@ package org.testcontainers.containers; import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.model.Container; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -24,15 +23,8 @@ import org.zeroturnaround.exec.stream.slf4j.Slf4jStream; import java.io.File; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -59,6 +51,8 @@ public class DockerComposeContainer> e private boolean pull = true; private boolean tailChildContainers; + private String project; + private final AtomicInteger nextAmbassadorPort = new AtomicInteger(2000); private final Map> ambassadorPortMappings = new ConcurrentHashMap<>(); private final SocatContainer ambassadorContainer = new SocatContainer(); @@ -94,6 +88,7 @@ public DockerComposeContainer(String identifier, List composeFiles) { // Use a unique identifier so that containers created for this compose environment can be identified this.identifier = identifier; + project = identifier + Base58.randomString(6).toLowerCase(); this.dockerClient = DockerClientFactory.instance().client(); } @@ -106,6 +101,7 @@ public void starting(Description description) { profiler.start("Docker Compose container startup"); synchronized (MUTEX) { + ResourceReaper.instance().registerFilterForCleanup("label=com.docker.compose.project=" + project); if (pull) { pullImages(); } @@ -114,7 +110,6 @@ public void starting(Description description) { if (tailChildContainers) { tailChildContainerLogs(); } - registerContainersForShutdown(); startAmbassadorContainers(profiler); } } @@ -142,9 +137,9 @@ private void tailChildContainerLogs() { private void runWithCompose(String cmd) { final DockerCompose dockerCompose; if (localCompose) { - dockerCompose = new LocalDockerCompose(composeFiles, identifier); + dockerCompose = new LocalDockerCompose(composeFiles, project); } else { - dockerCompose = new ContainerisedDockerCompose(composeFiles, identifier); + dockerCompose = new ContainerisedDockerCompose(composeFiles, project); } dockerCompose @@ -166,30 +161,7 @@ private void applyScaling() { } private void registerContainersForShutdown() { - // Ensure that all service containers that were launched by compose will be killed at shutdown - try { - final List containers = listChildContainers(); - - // register with ResourceReaper to ensure final shutdown with JVM - containers.forEach(container -> - ResourceReaper.instance().registerContainerForCleanup(container.getId(), container.getNames()[0])); - - // Compose can define their own networks as well; ensure these are cleaned up - dockerClient.listNetworksCmd().exec().forEach(network -> { - if (network.getName().contains(identifier)) { - spawnedNetworkIds.add(network.getId()); - ResourceReaper.instance().registerNetworkIdForCleanup(network.getId()); - } - }); - - // remember the IDs to allow containers to be killed as soon as we reach stop() - spawnedContainerIds.addAll(containers.stream() - .map(Container::getId) - .collect(Collectors.toSet())); - - } catch (DockerException e) { - logger().debug("Failed to stop a service container with exception", e); - } + ResourceReaper.instance().registerFilterForCleanup("label=com.docker.compose.project=" + project); } private List listChildContainers() { @@ -197,7 +169,7 @@ private List listChildContainers() { .withShowAll(true) .exec().stream() .filter(container -> Arrays.stream(container.getNames()).anyMatch(name -> - name.startsWith("/" + identifier))) + name.startsWith("/" + project))) .collect(toList()); } @@ -239,6 +211,8 @@ public void finished(Description description) { spawnedContainerIds.clear(); spawnedNetworkIds.clear(); + + project = identifier + Base58.randomString(6).toLowerCase(); } } diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 9cd915ffe35..dd02b00c717 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -211,10 +211,8 @@ private void tryStart(Profiler profiler) { profiler.start("Create container"); CreateContainerCmd createCommand = dockerClient.createContainerCmd(dockerImageName); applyConfiguration(createCommand); - createContainerCmdModifiers.forEach(hook -> hook.accept(createCommand)); containerId = createCommand.exec().getId(); - ResourceReaper.instance().registerContainerForCleanup(containerId, dockerImageName); logger().info("Starting container with ID: {}", containerId); profiler.start("Start container"); @@ -464,7 +462,12 @@ private void applyConfiguration(CreateContainerCmd createCommand) { createCommand.withPrivileged(privilegedMode); } - createCommand.withLabels(Collections.singletonMap("org.testcontainers", "true")); + createContainerCmdModifiers.forEach(hook -> hook.accept(createCommand)); + + Map labels = createCommand.getLabels(); + labels = new HashMap<>(labels != null ? labels : Collections.emptyMap()); + labels.putAll(DockerClientFactory.DEFAULT_LABELS); + createCommand.withLabels(labels); } private Set findLinksFromThisContainer(String alias, LinkableContainer linkableContainer) { diff --git a/core/src/main/java/org/testcontainers/containers/Network.java b/core/src/main/java/org/testcontainers/containers/Network.java index c2ea8a7b993..b71a74a455c 100644 --- a/core/src/main/java/org/testcontainers/containers/Network.java +++ b/core/src/main/java/org/testcontainers/containers/Network.java @@ -10,6 +10,8 @@ import org.testcontainers.utility.ResourceReaper; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -78,9 +80,12 @@ private String create() { consumer.accept(createNetworkCmd); } - String id = createNetworkCmd.exec().getId(); - ResourceReaper.instance().registerNetworkIdForCleanup(id); - return id; + Map labels = createNetworkCmd.getLabels(); + labels = new HashMap<>(labels != null ? labels : Collections.emptyMap()); + labels.putAll(DockerClientFactory.DEFAULT_LABELS); + createNetworkCmd.withLabels(labels); + + return createNetworkCmd.exec().getId(); } @Override diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index bac8772f842..7b24208a586 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -1,27 +1,53 @@ package org.testcontainers.utility; +import com.fasterxml.jackson.annotation.JsonProperty; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.exception.InternalServerErrorException; import com.github.dockerjava.api.exception.NotFoundException; +import com.github.dockerjava.api.model.AccessMode; +import com.github.dockerjava.api.model.Bind; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.Network; +import com.github.dockerjava.api.model.Ports; +import com.github.dockerjava.api.model.Volume; +import com.github.dockerjava.core.command.PullImageResultCallback; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.message.BasicNameValuePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.Socket; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Component that responsible for container removal and automatic cleanup of dead containers at JVM shutdown. */ +@Slf4j public final class ResourceReaper { private static final Logger LOGGER = LoggerFactory.getLogger(ResourceReaper.class); + + private static final List REGISTERED_FILTERS = new ArrayList<>(); + private static ResourceReaper instance; private final DockerClient dockerClient; private Map registeredContainers = new ConcurrentHashMap<>(); @@ -34,6 +60,94 @@ private ResourceReaper() { Runtime.getRuntime().addShutdownHook(new Thread(this::performCleanup)); } + @SneakyThrows(InterruptedException.class) + public static String start(String hostIpAddress, DockerClient client) { + String ryukImage = TestcontainersConfiguration.getInstance().getRyukImage(); + client.pullImageCmd(ryukImage).exec(new PullImageResultCallback()).awaitSuccess(); + + MountableFile mountableFile = MountableFile.forClasspathResource(ResourceReaper.class.getName().replace(".", "/") + ".class"); + + String ryukContainerId = client.createContainerCmd(ryukImage) + .withHostConfig(new HostConfig() { + @JsonProperty("AutoRemove") + boolean autoRemove = true; + }) + .withExposedPorts(new ExposedPort(8080)) + .withPublishAllPorts(true) + .withName("tc-ryuk-" + DockerClientFactory.SESSION_ID) + .withLabels(Collections.singletonMap(DockerClientFactory.TESTCONTAINERS_LABEL, "true")) + .withBinds(new Bind("/var/run/docker.sock", new Volume("/var/run/docker.sock"))) + .withBinds(new Bind(mountableFile.getResolvedPath(), new Volume("/dummy"), AccessMode.ro)) + .exec() + .getId(); + + client.startContainerCmd(ryukContainerId).exec(); + + InspectContainerResponse inspectedContainer = client.inspectContainerCmd(ryukContainerId).exec(); + + Integer ryukPort = inspectedContainer.getNetworkSettings().getPorts().getBindings().values().stream() + .flatMap(Stream::of) + .findFirst() + .map(Ports.Binding::getHostPortSpec) + .map(Integer::parseInt) + .get(); + + CountDownLatch ryukScheduledLatch = new CountDownLatch(1); + + REGISTERED_FILTERS.add( + URLEncodedUtils.format( + DockerClientFactory.DEFAULT_LABELS.entrySet().stream() + .map(it -> new BasicNameValuePair("label", it.getKey() + "=" + it.getValue())) + .collect(Collectors.toList()), + (String) null + ) + ); + + Thread kiraThread = new Thread( + () -> { + while (true) { + int index = 0; + try(Socket clientSocket = new Socket(hostIpAddress, ryukPort)) { + OutputStream out = clientSocket.getOutputStream(); + BufferedReader in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); + + while(true) { + if (REGISTERED_FILTERS.size() <= index) { + try { + TimeUnit.MILLISECONDS.sleep(100); + continue; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + out.write(REGISTERED_FILTERS.get(index).getBytes()); + out.write('\n'); + out.flush(); + + while (!"ACK".equalsIgnoreCase(in.readLine())) { + } + + ryukScheduledLatch.countDown(); + index++; + } + } catch (IOException e) { + log.warn("Can not connect to Ryuk at {}:{}", hostIpAddress, ryukPort, e); + } + } + }, + "tc-ryuk" + ); + kiraThread.setDaemon(true); + kiraThread.start(); + + // We need to wait before we can start any containers to make sure that we delete them + if (!ryukScheduledLatch.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("Can not connect to Ryuk"); + } + + return ryukContainerId; + } + public synchronized static ResourceReaper instance() { if (instance == null) { instance = new ResourceReaper(); @@ -51,6 +165,15 @@ public synchronized void performCleanup() { registeredNetworks.forEach(this::removeNetwork); } + /** + * Register a filter to be cleaned up. + * + * @param filter the filter + */ + public void registerFilterForCleanup(String filter) { + REGISTERED_FILTERS.add(filter); + } + /** * Register a container to be cleaned up, either on explicit call to stopAndRemoveContainer, or at JVM shutdown. * diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 890cd0d4279..5abcd4e6eb7 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -46,6 +46,10 @@ public String getTinyImage() { return (String) properties.getOrDefault("tinyimage.container.image", "alpine:3.5"); } + public String getRyukImage() { + return (String) properties.getOrDefault("ryuk.container.image", "bsideup/moby-ryuk:0.2.1"); + } + public boolean isDisableChecks() { return Boolean.parseBoolean((String) properties.getOrDefault("checks.disable", "false")); } From 69dcb17eadc02b3997e20dd23525ebc94ab25d42 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Mon, 15 Jan 2018 21:52:38 +0100 Subject: [PATCH 2/8] fix DockerComposeContainer.java --- .../org/testcontainers/containers/DockerComposeContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index 821f9382707..8ecd71e921a 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -101,7 +101,7 @@ public void starting(Description description) { profiler.start("Docker Compose container startup"); synchronized (MUTEX) { - ResourceReaper.instance().registerFilterForCleanup("label=com.docker.compose.project=" + project); + registerContainersForShutdown(); if (pull) { pullImages(); } From 23c084036f8650ba6dfd08543442ae131d88c385 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Mon, 15 Jan 2018 22:23:33 +0100 Subject: [PATCH 3/8] always encode filters, fix binds --- .../containers/DockerComposeContainer.java | 5 +- .../utility/ResourceReaper.java | 68 ++++++++++++------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index 8ecd71e921a..4ed046783e3 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -24,6 +24,7 @@ import java.io.File; import java.util.*; +import java.util.AbstractMap.SimpleEntry; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -161,7 +162,9 @@ private void applyScaling() { } private void registerContainersForShutdown() { - ResourceReaper.instance().registerFilterForCleanup("label=com.docker.compose.project=" + project); + ResourceReaper.instance().registerFilterForCleanup(Arrays.asList( + new SimpleEntry<>("label", "com.docker.compose.project=" + project) + )); } private List listChildContainers() { diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index 7b24208a586..58aff740916 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -27,6 +27,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.net.Socket; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -46,7 +47,7 @@ public final class ResourceReaper { private static final Logger LOGGER = LoggerFactory.getLogger(ResourceReaper.class); - private static final List REGISTERED_FILTERS = new ArrayList<>(); + private static final List>> REGISTERED_FILTERS = new ArrayList<>(); private static ResourceReaper instance; private final DockerClient dockerClient; @@ -76,8 +77,11 @@ public static String start(String hostIpAddress, DockerClient client) { .withPublishAllPorts(true) .withName("tc-ryuk-" + DockerClientFactory.SESSION_ID) .withLabels(Collections.singletonMap(DockerClientFactory.TESTCONTAINERS_LABEL, "true")) - .withBinds(new Bind("/var/run/docker.sock", new Volume("/var/run/docker.sock"))) - .withBinds(new Bind(mountableFile.getResolvedPath(), new Volume("/dummy"), AccessMode.ro)) + .withBinds( + new Bind("/var/run/docker.sock", new Volume("/var/run/docker.sock")), + // Not needed for Ryuk, but we perform pre-flight checks with it (micro optimization) + new Bind(mountableFile.getResolvedPath(), new Volume("/dummy"), AccessMode.ro) + ) .exec() .getId(); @@ -95,12 +99,9 @@ public static String start(String hostIpAddress, DockerClient client) { CountDownLatch ryukScheduledLatch = new CountDownLatch(1); REGISTERED_FILTERS.add( - URLEncodedUtils.format( - DockerClientFactory.DEFAULT_LABELS.entrySet().stream() - .map(it -> new BasicNameValuePair("label", it.getKey() + "=" + it.getValue())) - .collect(Collectors.toList()), - (String) null - ) + DockerClientFactory.DEFAULT_LABELS.entrySet().stream() + .>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue())) + .collect(Collectors.toList()) ); Thread kiraThread = new Thread( @@ -111,24 +112,36 @@ public static String start(String hostIpAddress, DockerClient client) { OutputStream out = clientSocket.getOutputStream(); BufferedReader in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); - while(true) { - if (REGISTERED_FILTERS.size() <= index) { - try { - TimeUnit.MILLISECONDS.sleep(100); - continue; - } catch (InterruptedException e) { - throw new RuntimeException(e); + synchronized (REGISTERED_FILTERS) { + while (true) { + if (REGISTERED_FILTERS.size() <= index) { + try { + REGISTERED_FILTERS.wait(1_000); + continue; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } - } - out.write(REGISTERED_FILTERS.get(index).getBytes()); - out.write('\n'); - out.flush(); + List> filters = REGISTERED_FILTERS.get(index); - while (!"ACK".equalsIgnoreCase(in.readLine())) { - } + String query = URLEncodedUtils.format( + filters.stream() + .map(it -> new BasicNameValuePair(it.getKey(), it.getValue())) + .collect(Collectors.toList()), + (String) null + ); + + log.debug("Sending '{}' to Ryuk", query); + out.write(query.getBytes()); + out.write('\n'); + out.flush(); - ryukScheduledLatch.countDown(); - index++; + while (!"ACK".equalsIgnoreCase(in.readLine())) { + } + + ryukScheduledLatch.countDown(); + index++; + } } } catch (IOException e) { log.warn("Can not connect to Ryuk at {}:{}", hostIpAddress, ryukPort, e); @@ -170,8 +183,11 @@ public synchronized void performCleanup() { * * @param filter the filter */ - public void registerFilterForCleanup(String filter) { - REGISTERED_FILTERS.add(filter); + public void registerFilterForCleanup(List> filter) { + synchronized (REGISTERED_FILTERS) { + REGISTERED_FILTERS.add(filter); + REGISTERED_FILTERS.notifyAll(); + } } /** From 8df3a488d9267a1531b2247d080926fa7e33ca4f Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Tue, 16 Jan 2018 08:33:25 +0100 Subject: [PATCH 4/8] use checkAndPullImage, add synchronized to the initial filter adding, update Ryuk image to 0.2.2 with timeout fix --- .../org/testcontainers/DockerClientFactory.java | 2 +- .../testcontainers/utility/ResourceReaper.java | 15 ++++++++------- .../utility/TestcontainersConfiguration.java | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/testcontainers/DockerClientFactory.java b/core/src/main/java/org/testcontainers/DockerClientFactory.java index efe322036d9..81050942f2d 100644 --- a/core/src/main/java/org/testcontainers/DockerClientFactory.java +++ b/core/src/main/java/org/testcontainers/DockerClientFactory.java @@ -177,7 +177,7 @@ private void checkMountableFile(DockerClient dockerClient, String id) { /** * Check whether the image is available locally and pull it otherwise */ - private void checkAndPullImage(DockerClient client, String image) { + public void checkAndPullImage(DockerClient client, String image) { List images = client.listImagesCmd().withImageNameFilter(image).exec(); if (images.isEmpty()) { client.pullImageCmd(image).exec(new PullImageResultCallback()).awaitSuccess(); diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index 58aff740916..e15b6f269ed 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -13,7 +13,6 @@ import com.github.dockerjava.api.model.Network; import com.github.dockerjava.api.model.Ports; import com.github.dockerjava.api.model.Volume; -import com.github.dockerjava.core.command.PullImageResultCallback; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.utils.URLEncodedUtils; @@ -64,7 +63,7 @@ private ResourceReaper() { @SneakyThrows(InterruptedException.class) public static String start(String hostIpAddress, DockerClient client) { String ryukImage = TestcontainersConfiguration.getInstance().getRyukImage(); - client.pullImageCmd(ryukImage).exec(new PullImageResultCallback()).awaitSuccess(); + DockerClientFactory.instance().checkAndPullImage(client, ryukImage); MountableFile mountableFile = MountableFile.forClasspathResource(ResourceReaper.class.getName().replace(".", "/") + ".class"); @@ -98,11 +97,13 @@ public static String start(String hostIpAddress, DockerClient client) { CountDownLatch ryukScheduledLatch = new CountDownLatch(1); - REGISTERED_FILTERS.add( - DockerClientFactory.DEFAULT_LABELS.entrySet().stream() - .>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue())) - .collect(Collectors.toList()) - ); + synchronized (REGISTERED_FILTERS) { + REGISTERED_FILTERS.add( + DockerClientFactory.DEFAULT_LABELS.entrySet().stream() + .>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue())) + .collect(Collectors.toList()) + ); + } Thread kiraThread = new Thread( () -> { diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 5abcd4e6eb7..d62728a4338 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -47,7 +47,7 @@ public String getTinyImage() { } public String getRyukImage() { - return (String) properties.getOrDefault("ryuk.container.image", "bsideup/moby-ryuk:0.2.1"); + return (String) properties.getOrDefault("ryuk.container.image", "bsideup/moby-ryuk:0.2.2"); } public boolean isDisableChecks() { From 29de66b426691da002f86fcf8c432dcb2aa2db57 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Tue, 16 Jan 2018 10:33:05 +0100 Subject: [PATCH 5/8] CR fixes, fix DockerClientFactoryTest, set shutdown hook lazily, rename REGISTERED_FILTERS -> DEATH_NOTE for more fun :) --- .../containers/DockerComposeContainer.java | 54 ++++++++++--------- .../utility/ResourceReaper.java | 36 ++++++++----- .../DockerClientFactoryTest.java | 37 +++++++------ 3 files changed, 73 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index 4ed046783e3..1e5ce4f076c 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -89,7 +89,7 @@ public DockerComposeContainer(String identifier, List composeFiles) { // Use a unique identifier so that containers created for this compose environment can be identified this.identifier = identifier; - project = identifier + Base58.randomString(6).toLowerCase(); + project = randomProjectId(); this.dockerClient = DockerClientFactory.instance().client(); } @@ -191,31 +191,33 @@ public void finished(Description description) { synchronized (MUTEX) { - // shut down the ambassador container - ambassadorContainer.stop(); - - // Kill the services using docker-compose try { - runWithCompose("down -v"); - - // If we reach here then docker-compose down has cleared networks and containers; - // we can unregister from ResourceReaper - spawnedContainerIds.forEach(ResourceReaper.instance()::unregisterContainer); - spawnedNetworkIds.forEach(ResourceReaper.instance()::unregisterNetwork); - } catch (Exception e) { - // docker-compose down failed; use ResourceReaper to ensure cleanup - - // kill the spawned service containers - spawnedContainerIds.forEach(ResourceReaper.instance()::stopAndRemoveContainer); - - // remove the networks after removing the containers - spawnedNetworkIds.forEach(ResourceReaper.instance()::removeNetworkById); + // shut down the ambassador container + ambassadorContainer.stop(); + + // Kill the services using docker-compose + try { + runWithCompose("down -v"); + + // If we reach here then docker-compose down has cleared networks and containers; + // we can unregister from ResourceReaper + spawnedContainerIds.forEach(ResourceReaper.instance()::unregisterContainer); + spawnedNetworkIds.forEach(ResourceReaper.instance()::unregisterNetwork); + } catch (Exception e) { + // docker-compose down failed; use ResourceReaper to ensure cleanup + + // kill the spawned service containers + spawnedContainerIds.forEach(ResourceReaper.instance()::stopAndRemoveContainer); + + // remove the networks after removing the containers + spawnedNetworkIds.forEach(ResourceReaper.instance()::removeNetworkById); + } + + spawnedContainerIds.clear(); + spawnedNetworkIds.clear(); + } finally { + project = randomProjectId(); } - - spawnedContainerIds.clear(); - spawnedNetworkIds.clear(); - - project = identifier + Base58.randomString(6).toLowerCase(); } } @@ -328,6 +330,10 @@ public SELF withTailChildContainers(boolean tailChildContainers) { private SELF self() { return (SELF) this; } + + private String randomProjectId() { + return identifier + Base58.randomString(6).toLowerCase(); + } } interface DockerCompose { diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index e15b6f269ed..fb7617fc7d4 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,18 +47,16 @@ public final class ResourceReaper { private static final Logger LOGGER = LoggerFactory.getLogger(ResourceReaper.class); - private static final List>> REGISTERED_FILTERS = new ArrayList<>(); + private static final List>> DEATH_NOTE = new ArrayList<>(); private static ResourceReaper instance; private final DockerClient dockerClient; private Map registeredContainers = new ConcurrentHashMap<>(); private Set registeredNetworks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private AtomicBoolean hookIsSet = new AtomicBoolean(false); private ResourceReaper() { dockerClient = DockerClientFactory.instance().client(); - - // If the JVM stops without containers being stopped, try and stop the container. - Runtime.getRuntime().addShutdownHook(new Thread(this::performCleanup)); } @SneakyThrows(InterruptedException.class) @@ -77,7 +76,7 @@ public static String start(String hostIpAddress, DockerClient client) { .withName("tc-ryuk-" + DockerClientFactory.SESSION_ID) .withLabels(Collections.singletonMap(DockerClientFactory.TESTCONTAINERS_LABEL, "true")) .withBinds( - new Bind("/var/run/docker.sock", new Volume("/var/run/docker.sock")), + new Bind("//var/run/docker.sock", new Volume("/var/run/docker.sock")), // Not needed for Ryuk, but we perform pre-flight checks with it (micro optimization) new Bind(mountableFile.getResolvedPath(), new Volume("/dummy"), AccessMode.ro) ) @@ -97,8 +96,8 @@ public static String start(String hostIpAddress, DockerClient client) { CountDownLatch ryukScheduledLatch = new CountDownLatch(1); - synchronized (REGISTERED_FILTERS) { - REGISTERED_FILTERS.add( + synchronized (DEATH_NOTE) { + DEATH_NOTE.add( DockerClientFactory.DEFAULT_LABELS.entrySet().stream() .>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue())) .collect(Collectors.toList()) @@ -113,17 +112,17 @@ public static String start(String hostIpAddress, DockerClient client) { OutputStream out = clientSocket.getOutputStream(); BufferedReader in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); - synchronized (REGISTERED_FILTERS) { + synchronized (DEATH_NOTE) { while (true) { - if (REGISTERED_FILTERS.size() <= index) { + if (DEATH_NOTE.size() <= index) { try { - REGISTERED_FILTERS.wait(1_000); + DEATH_NOTE.wait(1_000); continue; } catch (InterruptedException e) { throw new RuntimeException(e); } } - List> filters = REGISTERED_FILTERS.get(index); + List> filters = DEATH_NOTE.get(index); String query = URLEncodedUtils.format( filters.stream() @@ -185,9 +184,9 @@ public synchronized void performCleanup() { * @param filter the filter */ public void registerFilterForCleanup(List> filter) { - synchronized (REGISTERED_FILTERS) { - REGISTERED_FILTERS.add(filter); - REGISTERED_FILTERS.notifyAll(); + synchronized (DEATH_NOTE) { + DEATH_NOTE.add(filter); + DEATH_NOTE.notifyAll(); } } @@ -198,6 +197,7 @@ public void registerFilterForCleanup(List> filter) { * @param imageName the image name of the container (used for logging) */ public void registerContainerForCleanup(String containerId, String imageName) { + setHook(); registeredContainers.put(containerId, imageName); } @@ -273,6 +273,7 @@ private void stopContainer(String containerId, String imageName) { * @param id the ID of the network */ public void registerNetworkIdForCleanup(String id) { + setHook(); registeredNetworks.add(id); } @@ -346,4 +347,11 @@ public void unregisterNetwork(String identifier) { public void unregisterContainer(String identifier) { registeredContainers.remove(identifier); } + + private void setHook() { + if (hookIsSet.compareAndSet(false, true)) { + // If the JVM stops without containers being stopped, try and stop the container. + Runtime.getRuntime().addShutdownHook(new Thread(this::performCleanup)); + } + } } diff --git a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java index a6cb304d4e3..e13c3875f31 100644 --- a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java +++ b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java @@ -1,5 +1,6 @@ package org.testcontainers; +import com.github.dockerjava.api.exception.NotFoundException; import org.junit.Test; import org.rnorth.visibleassertions.VisibleAssertions; import org.testcontainers.DockerClientFactory.DiskSpaceUsage; @@ -11,24 +12,28 @@ */ public class DockerClientFactoryTest { - @Test - public void runCommandInsideDockerShouldNotFailIfImageDoesNotExistsLocally() { + @Test + public void runCommandInsideDockerShouldNotFailIfImageDoesNotExistsLocally() { - final DockerClientFactory dockFactory = DockerClientFactory.instance(); - //remove tiny image, so it will be pulled during next command run - dockFactory.client() - .removeImageCmd(TestcontainersConfiguration.getInstance().getTinyImage()) - .withForce(true).exec(); + final DockerClientFactory dockFactory = DockerClientFactory.instance(); + try { + //remove tiny image, so it will be pulled during next command run + dockFactory.client() + .removeImageCmd(TestcontainersConfiguration.getInstance().getTinyImage()) + .withForce(true).exec(); + } catch (NotFoundException ignored) { + // Do not fail if it's not pulled yet + } - dockFactory.runInsideDocker( - cmd -> cmd.withCmd("sh", "-c", "echo 'SUCCESS'"), - (client, id) -> - client.logContainerCmd(id) - .withStdOut(true) - .exec(new LogToStringContainerCallback()) - .toString() - ); - } + dockFactory.runInsideDocker( + cmd -> cmd.withCmd("sh", "-c", "echo 'SUCCESS'"), + (client, id) -> + client.logContainerCmd(id) + .withStdOut(true) + .exec(new LogToStringContainerCallback()) + .toString() + ); + } @Test public void shouldHandleBigDiskSize() throws Exception { From dbfc5bb9bf8abf6e063525984cf5226dd803a00c Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Tue, 16 Jan 2018 12:05:39 +0100 Subject: [PATCH 6/8] CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54f50a67436..72b8899e1df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Changed - Added Kafka module ([\#546](https://github.com/testcontainers/testcontainers-java/pull/546)) +- Added "Death Note" to track & kill spawned containers even if the JVM was "kill -9"ed ([\#545](https://github.com/testcontainers/testcontainers-java/pull/545)) ## [1.5.1] - 2017-12-19 From cf60eba3b74ef675345da82a13012b8216c6c6f7 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Mon, 22 Jan 2018 08:40:45 +0100 Subject: [PATCH 7/8] expand "tc" to "testcontainers", log the purpose of Ryuk --- .../src/main/java/org/testcontainers/DockerClientFactory.java | 2 +- .../main/java/org/testcontainers/utility/ResourceReaper.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/testcontainers/DockerClientFactory.java b/core/src/main/java/org/testcontainers/DockerClientFactory.java index 81050942f2d..a2177c886a6 100644 --- a/core/src/main/java/org/testcontainers/DockerClientFactory.java +++ b/core/src/main/java/org/testcontainers/DockerClientFactory.java @@ -114,7 +114,7 @@ public DockerClient client() { " Total Memory: " + dockerInfo.getMemTotal() / (1024 * 1024) + " MB"); String ryukContainerId = ResourceReaper.start(hostIpAddress, client); - log.info("Ryuk started"); + log.info("Ryuk started - will monitor and terminate Testcontainers containers on JVM exit"); VisibleAssertions.info("Checking the system..."); diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index fb7617fc7d4..6e169114e57 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -73,7 +73,7 @@ public static String start(String hostIpAddress, DockerClient client) { }) .withExposedPorts(new ExposedPort(8080)) .withPublishAllPorts(true) - .withName("tc-ryuk-" + DockerClientFactory.SESSION_ID) + .withName("testcontainers-ryuk-" + DockerClientFactory.SESSION_ID) .withLabels(Collections.singletonMap(DockerClientFactory.TESTCONTAINERS_LABEL, "true")) .withBinds( new Bind("//var/run/docker.sock", new Volume("/var/run/docker.sock")), @@ -148,7 +148,7 @@ public static String start(String hostIpAddress, DockerClient client) { } } }, - "tc-ryuk" + "testcontainers-ryuk" ); kiraThread.setDaemon(true); kiraThread.start(); From 0ad279734cc81bdd89669f23e22c396555e21919 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sat, 27 Jan 2018 12:05:38 +0100 Subject: [PATCH 8/8] fix after merge --- .../testcontainers/containers/DockerComposeContainer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index 1e5ce4f076c..ecc91dfe0a2 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -25,8 +25,9 @@ import java.io.File; import java.util.*; import java.util.AbstractMap.SimpleEntry; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -245,7 +246,7 @@ public SELF withExposedService(String serviceName, int servicePort) { int ambassadorPort = nextAmbassadorPort.getAndIncrement(); ambassadorPortMappings.computeIfAbsent(serviceName, __ -> new ConcurrentHashMap<>()).put(servicePort, ambassadorPort); ambassadorContainer.withTarget(ambassadorPort, serviceName, servicePort); - ambassadorContainer.addLink(new FutureContainer(this.identifier + "_" + serviceName), serviceName); + ambassadorContainer.addLink(new FutureContainer(this.project + "_" + serviceName), serviceName); return self(); }