Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement conditional depends_on #453

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 106 additions & 10 deletions podman_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,35 @@ def strverscmp_lt(a, b):
return a_ls < b_ls


class DependsCondition: # pylint: disable=too-few-public-methods
# enum for possible types of depends_on conditions
# see https://github.com/compose-spec/compose-spec/blob/master/spec.md#long-syntax-1
STARTED = 0
HEALTHY = 1
COMPLETED = 2

@classmethod
def to_enum(cls, condition):
"""
Converts and returns a condition value into a valid enum value.
"""
if condition == "service_healthy":
return cls.HEALTHY
if condition == "service_completed_successfully":
return cls.COMPLETED
# use cls.STARTED as a catch-all value even
# if the condition value is not within spec
return cls.STARTED


def wait(func):
def wrapper(*args, **kwargs):
while not func(*args, **kwargs):
time.sleep(0.5)

return wrapper


def parse_short_mount(mount_str, basedir):
mount_a = mount_str.split(":")
mount_opt_dict = {}
Expand Down Expand Up @@ -987,25 +1016,46 @@ def flat_deps(services, with_extends=False):
create dependencies "_deps" or update it recursively for all services
"""
for name, srv in services.items():
deps = set()
srv["_deps"] = deps
deps = {}
if with_extends:
ext = srv.get("extends", {}).get("service", None)
if ext:
if ext != name:
deps.add(ext)
deps[ext] = DependsCondition.STARTED
continue
# NOTE: important that the get call is kept as-is, since depends_on
# can be an empty string and in that case we want to have an empty list
deps_ls = srv.get("depends_on", None) or []
if is_str(deps_ls):
deps_ls = [deps_ls]
# depends_on: "foo"
# treat as condition: service_started
deps_ls = {deps_ls: DependsCondition.STARTED}
elif is_dict(deps_ls):
deps_ls = list(deps_ls.keys())
deps.update(deps_ls)
# depends_on:
# foo:
# condition: service_xxx
tmp = {}
for service, condition in deps_ls.items():
condition = DependsCondition.to_enum(condition.get("condition"))
tmp[service] = condition
deps_ls = tmp
else:
# depends_on:
# - foo
# treat as condition: service_started
deps_ls = {dep: DependsCondition.STARTED for dep in deps_ls}
deps = {**deps, **deps_ls}
# parse link to get service name and remove alias
# NOTE: important that the get call is kept as-is, since links can
# be an empty string and in that case we want to have an empty list
links_ls = srv.get("links", None) or []
if not is_list(links_ls):
links_ls = [links_ls]
deps.update([(c.split(":")[0] if ":" in c else c) for c in links_ls])
deps = {
**deps,
**{c.split(":")[0]: DependsCondition.STARTED for c in links_ls},
}
srv["_deps"] = deps
for name, srv in services.items():
rec_deps(services, name)

Expand Down Expand Up @@ -1045,7 +1095,7 @@ def run(
podman_args,
cmd="",
cmd_args=None,
wait=True,
_wait=True,
sleep=1,
obj=None,
log_formatter=None,
Expand All @@ -1071,7 +1121,7 @@ def run(
else:
p = subprocess.Popen(cmd_ls) # pylint: disable=consider-using-with

if wait:
if _wait:
exit_code = p.wait()
log("exit code:", exit_code)
if obj is not None:
Expand Down Expand Up @@ -1922,12 +1972,55 @@ def get_excluded(compose, args):
if args.services:
excluded = set(compose.services)
for service in args.services:
excluded -= compose.services[service]["_deps"]
excluded -= set(compose.services[service]["_deps"].keys())
excluded.discard(service)
log("** excluding: ", excluded)
return excluded


@wait
def wait_healthy(compose, container_name):
info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]

if not info["Config"].get("Healthcheck"):
raise ValueError("Container %s does not define a health check" % container_name)

health = info["State"]["Healthcheck"]["Status"]
Elkasitu marked this conversation as resolved.
Show resolved Hide resolved
if health == "unhealthy":
raise RuntimeError(
"Container %s is in unhealthy state, aborting" % container_name
)
return health == "healthy"


@wait
def wait_completed(compose, container_name):
info = json.loads(compose.podman.output([], "inspect", [container_name]))[0]

if info["State"]["Status"] == "exited":
exit_code = info["State"]["ExitCode"]
if exit_code != 0:
raise RuntimeError(
"Container %s didn't complete successfully, exit code: %d"
% (container_name, exit_code)
)
return True
return False


def wait_for_dependencies(compose, container):
for dep, condition in container["_deps"].items():
dep_container_name = compose.container_names_by_service[dep][0]
if condition == DependsCondition.STARTED:
# ignore -- will be handled by container order
continue
if condition == DependsCondition.HEALTHY:
wait_healthy(compose, dep_container_name)
else:
# implies DependsCondition.COMPLETED
wait_completed(compose, dep_container_name)


@cmd_run(
podman_compose, "up", "Create and start the entire stack or some of its services"
)
Expand Down Expand Up @@ -1970,6 +2063,8 @@ def compose_up(compose, args):
log("** skipping: ", cnt["name"])
continue
podman_args = container_to_args(compose, cnt, detached=args.detach)
if podman_command == "run":
wait_for_dependencies(compose, cnt)
subproc = compose.podman.run([], podman_command, podman_args)
if podman_command == "run" and subproc and subproc.returncode:
compose.podman.run([], "start", [cnt["name"]])
Expand Down Expand Up @@ -2005,6 +2100,7 @@ def compose_up(compose, args):
continue
# TODO: remove sleep from podman.run
obj = compose if exit_code_from == cnt["_service"] else None
wait_for_dependencies(compose, cnt)
thread = Thread(
target=compose.podman.run,
args=[[], "start", ["-a", cnt["name"]]],
Expand Down
37 changes: 37 additions & 0 deletions pytests/test_dependencies.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import pytest

from podman_compose import flat_deps, DependsCondition


@pytest.fixture
def basic_services():
return {
"foo": {},
"bar": {
# string dependency
"depends_on": "foo",
},
"baz": {
# list dependency
"depends_on": ["bar"],
},
"ham": {
# dict / conditional dependency
"depends_on": {
"foo": {
"condition": "service_healthy",
},
},
},
}


def test_flat_deps(basic_services):
flat_deps(basic_services)
assert basic_services["foo"]["_deps"] == {}
assert basic_services["bar"]["_deps"] == {"foo": DependsCondition.STARTED}
assert basic_services["baz"]["_deps"] == {
"bar": DependsCondition.STARTED,
"foo": DependsCondition.STARTED,
}
assert basic_services["ham"]["_deps"] == {"foo": DependsCondition.HEALTHY}
56 changes: 55 additions & 1 deletion tests/deps/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,26 @@ services:
tmpfs:
- /run
- /tmp
healthcheck:
# test that httpd is running, the brackets [] thing is a trick
# to ignore the grep process returned by ps, meaning that this
# should only be true if httpd is currently running
test: ps | grep "[h]ttpd"
interval: 10s
timeout: 5s
retries: 5
sleep:
image: busybox
command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
depends_on: "web"
tmpfs:
- /run
- /tmp
healthcheck:
test: sleep 15
interval: 10s
timeout: 20s
retries: 5
sleep2:
image: busybox
command: ["/bin/busybox", "sh", "-c", "sleep 3600"]
Expand All @@ -21,4 +34,45 @@ services:
tmpfs:
- /run
- /tmp

setup:
image: busybox
command: ["/bin/busybox", "sh", "-c", "sleep 30"]
tmpfs:
- /run
- /tmp
wait_started:
image: busybox
command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
depends_on:
sleep:
condition: service_started
tmpfs:
- /run
- /tmp
wait_healthy:
image: busybox
command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
depends_on:
web:
condition: service_healthy
tmpfs:
- /run
- /tmp
wait_multiple_healthchecks:
image: busybox
command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
depends_on:
sleep:
condition: service_healthy
tmpfs:
- /run
- /tmp
wait_completed_successfully:
image: busybox
command: ["/bin/busybox", "sh", "-c", "echo 'hello world'"]
depends_on:
setup:
condition: service_completed_successfully
tmpfs:
- /run
- /tmp