From b50bdb94aee25c2b49726c31d2bc9be962873b78 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Oct 2025 00:04:02 +0000 Subject: [PATCH] Configure Celery for graceful shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements soft shutdown feature from Celery 5.5.3 to prevent task interruption during pod termination. Resolves #5000. Changes: - Add worker_soft_shutdown_timeout (28s) to Celery config in settings.py - Add REMAP_SIGTERM=SIGQUIT to K8s shared env vars to trigger soft shutdown When K8s sends SIGTERM during pod termination, workers will now: 1. Stop accepting new tasks 2. Continue processing current task for up to 28 seconds 3. Exit cleanly if task completes, or timeout after 28s 4. Allow K8s 2s buffer before 30s grace period expires 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- contentcuration/contentcuration/settings.py | 5 +++++ docker-compose.yml | 1 + 2 files changed, 6 insertions(+) diff --git a/contentcuration/contentcuration/settings.py b/contentcuration/contentcuration/settings.py index 0f18ed0131..285e7bef76 100644 --- a/contentcuration/contentcuration/settings.py +++ b/contentcuration/contentcuration/settings.py @@ -348,6 +348,11 @@ def gettext(s): "result_serializer": "json", "result_extended": True, "worker_send_task_events": True, + # Graceful shutdown: allow 28 seconds for tasks to complete before forced termination + # This is 2 seconds less than Kubernetes terminationGracePeriodSeconds (30s) + "worker_soft_shutdown_timeout": int( + os.getenv("CELERY_WORKER_SOFT_SHUTDOWN_TIMEOUT", "28") + ), } # When cleaning up orphan nodes, only clean up any that have been last modified diff --git a/docker-compose.yml b/docker-compose.yml index 68bb5e2500..719fc797ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ x-studio-environment: CELERY_BROKER_ENDPOINT: redis CELERY_RESULT_BACKEND_ENDPOINT: redis CELERY_REDIS_PASSWORD: "" + REMAP_SIGTERM: "SIGQUIT" PROBER_STUDIO_BASE_URL: http://studio-app:8080/{path} x-studio-worker: