From 65035b20e804b2a24c7ad73de7cd4b050c7b9cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 2 Jun 2026 20:29:37 +0000 Subject: [PATCH] ci: treat docker container-removal conflict as flaky MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NeMo-Run DockerExecutor can attempt to remove the same container twice during cleanup, producing a harmless `409 Conflict "removal of container ... is already in progress"` race. On an otherwise-successful job this noise should never trip or pollute CI. Classify the docker-removal-conflict signatures as flaky in `is_flaky_failure` so the launcher retries instead of failing fatally. The fix lives in the launcher because nemo_run's `docker.py` is an external dependency. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: oliver könig --- tests/test_utils/python_scripts/launch_nemo_run_workload.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_utils/python_scripts/launch_nemo_run_workload.py b/tests/test_utils/python_scripts/launch_nemo_run_workload.py index 0597a3189da..80eaa51edee 100644 --- a/tests/test_utils/python_scripts/launch_nemo_run_workload.py +++ b/tests/test_utils/python_scripts/launch_nemo_run_workload.py @@ -42,6 +42,9 @@ def is_flaky_failure(concat_allranks_logs: str) -> bool: or "free(): corrupted unsorted chunks" in concat_allranks_logs or "Segfault encountered" in concat_allranks_logs or "The following metrics failed" in concat_allranks_logs + or "removal of container" in concat_allranks_logs + or "is already in progress" in concat_allranks_logs + or "Error deleting container" in concat_allranks_logs )