Wait for depends_on conditions before starting containers

This commit implements the long-syntax / conditional depends_on compose
mechanism which instructs the compose system to wait for a certain
condition before starting a container.

Currently available conditions are:

- service_started: same behavior as before this commit, the depending
  container will start as soon as the depended on container has started
- service_healthy: if the depended on container has a healthcheck, wait
  until said container is marked as healthy before starting the
  depending container
- service_completed_successfully: wait until the depended on container
  has exited and its exit code is 0, after which the depending container
  can be started

This mechanism is part of the v3 [1] compose spec and is useful for
controlling container startup based on other containers that can take a
certain amount of time to start or on containers that do complicated
setups and must exit before starting other containers.

[1] https://red.ht/conditional-depends

Signed-off-by: Adrian Torres <[email protected]>

podman_compose.py

@@ -123,6 +123,14 @@ def to_enum(cls, condition):
         return cls.STARTED
 
 
+def wait(func):
+    def wrapper(*args, **kwargs):
+        while not func(*args, **kwargs):
+            time.sleep(0.5)
+
+    return wrapper
+
+
 def parse_short_mount(mount_str, basedir):
     mount_a = mount_str.split(":")
     mount_opt_dict = {}
@@ -1087,7 +1095,7 @@ def run(
         podman_args,
         cmd="",
         cmd_args=None,
-        wait=True,
+        _wait=True,
         sleep=1,
         obj=None,
         log_formatter=None,
@@ -1113,7 +1121,7 @@ def run(
         else:
             p = subprocess.Popen(cmd_ls)  # pylint: disable=consider-using-with
 
-        if wait:
+        if _wait:
             exit_code = p.wait()
             log("exit code:", exit_code)
             if obj is not None:
@@ -1970,6 +1978,49 @@ def get_excluded(compose, args):
     return excluded
 
 
+@wait
+def wait_healthy(compose, container_name):
+    info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]
+
+    if not info["Config"].get("Healthcheck"):
+        raise ValueError("Container %s does not define a health check" % container_name)
+
+    health = info["State"]["Healthcheck"]["Status"]
+    if health == "unhealthy":
+        raise RuntimeError(
+            "Container %s is in unhealthy state, aborting" % container_name
+        )
+    return health == "healthy"
+
+
+@wait
+def wait_completed(compose, container_name):
+    info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]
+
+    if info["State"]["Status"] == "exited":
+        exit_code = info["State"]["ExitCode"]
+        if exit_code != 0:
+            raise RuntimeError(
+                "Container %s didn't complete successfully, exit code: %d"
+                % (container_name, exit_code)
+            )
+        return True
+    return False
+
+
+def wait_for_dependencies(compose, container):
+    for dep, condition in container["_deps"].items():
+        dep_container_name = compose.container_names_by_service[dep][0]
+        if condition == DependsCondition.STARTED:
+            # ignore -- will be handled by container order
+            continue
+        if condition == DependsCondition.HEALTHY:
+            wait_healthy(compose, dep_container_name)
+        else:
+            # implies DependsCondition.COMPLETED
+            wait_completed(compose, dep_container_name)
+
+
 @cmd_run(
     podman_compose, "up", "Create and start the entire stack or some of its services"
 )
@@ -2012,6 +2063,8 @@ def compose_up(compose, args):
             log("** skipping: ", cnt["name"])
             continue
         podman_args = container_to_args(compose, cnt, detached=args.detach)
+        if podman_command == "run":
+            wait_for_dependencies(compose, cnt)
         subproc = compose.podman.run([], podman_command, podman_args)
         if podman_command == "run" and subproc and subproc.returncode:
             compose.podman.run([], "start", [cnt["name"]])
@@ -2047,6 +2100,7 @@ def compose_up(compose, args):
             continue
         # TODO: remove sleep from podman.run
         obj = compose if exit_code_from == cnt["_service"] else None
+        wait_for_dependencies(compose, cnt)
         thread = Thread(
             target=compose.podman.run,
             args=[[], "start", ["-a", cnt["name"]]],

tests/deps/docker-compose.yaml

@@ -6,13 +6,26 @@ services:
       tmpfs:
         - /run
         - /tmp
+      healthcheck:
+        # test that httpd is running, the brackets [] thing is a trick
+        # to ignore the grep process returned by ps, meaning that this
+        # should only be true if httpd is currently running
+        test: ps | grep "[h]ttpd"
+        interval: 10s
+        timeout: 5s
+        retries: 5
     sleep:
       image: busybox
       command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
       depends_on: "web"
       tmpfs:
         - /run
         - /tmp
+      healthcheck:
+        test: sleep 15
+        interval: 10s
+        timeout: 20s
+        retries: 5
     sleep2:
       image: busybox
       command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
@@ -21,7 +34,13 @@ services:
       tmpfs:
         - /run
         - /tmp
-    hello_world:
+    setup:
+      image: busybox
+      command: ["/bin/busybox", "sh", "-c", "sleep 30"]
+      tmpfs:
+        - /run
+        - /tmp
+    wait_started:
       image: busybox
       command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
       depends_on:
@@ -30,12 +49,16 @@ services:
       tmpfs:
         - /run
         - /tmp
-      healthcheck:
-        test: echo "hello world"
-        interval: 10s
-        timeout: 5s
-        retries: 5
-    hello_world_2:
+    wait_healthy:
+      image: busybox
+      command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
+      depends_on:
+        web:
+          condition: service_healthy
+      tmpfs:
+        - /run
+        - /tmp
+    wait_multiple_healthchecks:
       image: busybox
       command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
       depends_on:
@@ -44,3 +67,12 @@ services:
       tmpfs:
         - /run
         - /tmp
+    wait_completed_successfully:
+      image: busybox
+      command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
+      depends_on:
+        setup:
+          condition: service_completed_successfully
+      tmpfs:
+        - /run
+        - /tmp