From 4cd984aad2bb174aca198f62a767b84a8d31f11c Mon Sep 17 00:00:00 2001 From: pnkcaht Date: Mon, 26 Jan 2026 11:19:54 -0500 Subject: [PATCH 1/4] test: reproduce docker container not killed on agent shutdown --- .woodpecker.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .woodpecker.yml diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 00000000000..8469ce8a5d8 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,9 @@ +skip_clone: true + +steps: + sleep_30s: + image: bash + commands: + - echo wait 30s + - sleep 30s + - echo wait done From 252bfd61e4825d416e5d9010e6cc9b7495f6339f Mon Sep 17 00:00:00 2001 From: pnkcaht Date: Mon, 26 Jan 2026 11:29:50 -0500 Subject: [PATCH 2/4] fix(docker): kill running container when step context is canceled --- pipeline/backend/docker/docker.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index 8a48fdd72e6..ee22694cad2 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -250,11 +250,29 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str return e.client.ContainerStart(ctx, containerName, container.StartOptions{}) } +// WaitStep waits for a step container to exit. +// +// When the context is canceled, the container is immediately killed to prevent +// orphaned containers from continuing to run after agent shutdown. func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID string) (*backend.State, error) { - log := log.Logger.With().Str("taskUUID", taskUUID).Str("stepUUID", step.UUID).Logger() + log := log.Logger.With(). + Str("taskUUID", taskUUID). + Str("stepUUID", step.UUID). + Logger() + log.Trace().Msgf("wait for step %s", step.Name) containerName := toContainerName(step) + done := make(chan struct{}) + + // Ensure container is killed if context is canceled (SIGTERM / pipeline cancel) + go func() { + select { + case <-ctx.Done(): + _ = e.client.ContainerKill(context.Background(), containerName, "9") + case <-done: + } + }() wait, errC := e.client.ContainerWait(ctx, containerName, "") select { @@ -264,6 +282,9 @@ func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID stri log.Trace().Msgf("ContainerWait returned with err: %v", err) } + // Stop cancellation watcher + close(done) + info, err := e.client.ContainerInspect(ctx, containerName) if err != nil { return nil, err From f7d6b9d99726bd8aef74a7e88f389dfb2fa2e38b Mon Sep 17 00:00:00 2001 From: pnkcaht Date: Mon, 26 Jan 2026 11:31:03 -0500 Subject: [PATCH 3/4] fix(docker): kill running container when step context is canceled --- .woodpecker.yml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .woodpecker.yml diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index 8469ce8a5d8..00000000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,9 +0,0 @@ -skip_clone: true - -steps: - sleep_30s: - image: bash - commands: - - echo wait 30s - - sleep 30s - - echo wait done From 1b355686206741dbb8fc941c12243f65c3142957 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 27 Jan 2026 00:10:17 +0100 Subject: [PATCH 4/4] Apply suggestion from @6543 --- pipeline/backend/docker/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index ee22694cad2..d51eb351d70 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -269,7 +269,7 @@ func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID stri go func() { select { case <-ctx.Done(): - _ = e.client.ContainerKill(context.Background(), containerName, "9") + _ = e.client.ContainerKill(context.Background(), containerName, "9") //nolint:contextcheck case <-done: } }()