From e32c6d24b4bf083a2c4ae13373c427a2ee352bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 2 Jun 2026 20:27:30 +0000 Subject: [PATCH 1/2] ci: make CI resilient to pip/uv network timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host-side actions/setup-python "Upgrading pip..." step downloads the latest pip from PyPI on a fresh toolcache and aborts the whole job on a transient read timeout (~15s default, no retries). The same flakiness hits in-container uv/pip downloads. - cicd-main.yml: add PIP_DEFAULT_TIMEOUT=120 and PIP_RETRIES=5 to the three job-level env blocks so the host setup-python pip upgrade retries instead of hard-failing. - action.yml: export PIP_DEFAULT_TIMEOUT / PIP_RETRIES / UV_HTTP_TIMEOUT in both generated run-scripts so in-container uv sync / pip get the same cushion. - launch_nemo_run_workload.py: treat read/connection timeouts as flaky so the launcher auto-restarts, and collect harness/task logs (not just per-rank std*.log) so those signatures are actually seen. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: oliver könig --- .github/actions/action.yml | 6 +++ .github/workflows/cicd-main.yml | 6 +++ .../launch_nemo_run_workload.py | 43 ++++++++++++++++--- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 48e272bcf34..b4261cf1ce9 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -127,6 +127,9 @@ runs: export PYTHONPATH=$(pwd) export NEMORUN_HOME=$(pwd) export NCCL_DEBUG=INFO + export PIP_DEFAULT_TIMEOUT=120 + export PIP_RETRIES=5 + export UV_HTTP_TIMEOUT=120 uv venv .venv uv cache clean uv sync --no-cache --only-group test @@ -169,6 +172,9 @@ runs: export PYTHONPATH=$(pwd) export NEMORUN_HOME=$(pwd) + export PIP_DEFAULT_TIMEOUT=120 + export PIP_RETRIES=5 + export UV_HTTP_TIMEOUT=120 uv venv .venv uv cache clean uv sync --no-cache --only-group test diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index d7a8c92331e..d45b43ee316 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -716,6 +716,8 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 steps: - name: Checkout uses: actions/checkout@v6 @@ -865,6 +867,8 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 if: | !cancelled() && needs.cicd-integration-gate.outputs.should_run == 'true' @@ -962,6 +966,8 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 if: | !cancelled() && needs.cicd-integration-gate.outputs.should_run == 'true' diff --git a/tests/test_utils/python_scripts/launch_nemo_run_workload.py b/tests/test_utils/python_scripts/launch_nemo_run_workload.py index 0597a3189da..159417fa9c7 100644 --- a/tests/test_utils/python_scripts/launch_nemo_run_workload.py +++ b/tests/test_utils/python_scripts/launch_nemo_run_workload.py @@ -38,6 +38,11 @@ def is_flaky_failure(concat_allranks_logs: str) -> bool: or "zmq.error.ZMQError: Address already in use" in concat_allranks_logs or "We couldn't connect to 'https://huggingface.co'" in concat_allranks_logs or "Unpack failed: incomplete input" in concat_allranks_logs + or "The read operation timed out" in concat_allranks_logs + or "Read timed out" in concat_allranks_logs + or "TimeoutError" in concat_allranks_logs + or "Connection broken" in concat_allranks_logs + or "Temporary failure in name resolution" in concat_allranks_logs or "unspecified launch failure" in concat_allranks_logs or "free(): corrupted unsorted chunks" in concat_allranks_logs or "Segfault encountered" in concat_allranks_logs @@ -45,6 +50,38 @@ def is_flaky_failure(concat_allranks_logs: str) -> bool: ) +def _collect_failure_logs(workdir: pathlib.Path) -> list[str]: + """Reads every log file that may carry a flaky-failure signature. + + The per-rank ``attempt_0/*/std*.log`` files only contain torchrun training + output. The golden-value comparison runs in ``run_ci_test.sh`` and emits its + assertion (e.g. ``The following metrics failed``) to the nemo-run task log, + which lives outside that per-rank tree. Globbing both ensures harness-side + failures are seen by ``is_flaky_failure`` and therefore eligible for retry. + + Args: + workdir: Working directory under which nemo-run writes all log files. + + Returns: + The concatenated lines of every discovered log file, deduplicated by + resolved path to avoid double-counting overlapping globs. + """ + seen_paths = set() + collected_lines: list[str] = [] + for pattern in ("**/attempt_0/*/std*.log", "**/*.log"): + for log_file_path in workdir.glob(pattern): + resolved = log_file_path.resolve() + if resolved in seen_paths or not log_file_path.is_file(): + continue + seen_paths.add(resolved) + try: + with open(log_file_path, "r", errors="replace") as f: + collected_lines.extend(f.readlines()) + except OSError as error: + logger.warning("Could not read log file %s: %s", log_file_path, error) + return collected_lines + + @click.command() @click.option("--scope", required=True, type=str, help="Scope of the workload") @click.option("--model", required=True, type=str, help="Model of the workload") @@ -194,12 +231,8 @@ def __getattr__(self, name): sys.exit(0) logger.error(f"Job failed with status: {job_dict['status']}") - log_file_paths = pathlib.Path(os.getcwd()).glob("**/attempt_0/*/std*.log") all_ranks_all_logs = [tee_buffer.getvalue()] - for log_file_path in log_file_paths: - with open(log_file_path, "r") as f: - all_logs = f.readlines() - all_ranks_all_logs.extend(all_logs) + all_ranks_all_logs.extend(_collect_failure_logs(pathlib.Path(os.getcwd()))) all_ranks_all_logs_string = "\n".join(all_ranks_all_logs) if is_flaky_failure(all_ranks_all_logs_string): logger.warning("Detected flaky failure, attempt restart.") From 3a23eb605434f2d88d04f46f7e4de6efa679e662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Tue, 2 Jun 2026 20:34:18 +0000 Subject: [PATCH 2/2] ci: retry manylinux image pull in wheel build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The megatron-fsdp wheel build failed when `docker run` implicitly pulled quay.io/pypa/manylinux_2_28_x86_64 and the registry request timed out: `net/http: request canceled while waiting for connection`. Pre-pull the image with the repo's standard 3x retry loop so a transient quay.io timeout no longer fails the job; subsequent docker runs reuse the cached image. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: oliver könig --- .github/workflows/_build_test_publish_wheel.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/_build_test_publish_wheel.yml b/.github/workflows/_build_test_publish_wheel.yml index 0df8756d082..9e37e068b6d 100644 --- a/.github/workflows/_build_test_publish_wheel.yml +++ b/.github/workflows/_build_test_publish_wheel.yml @@ -72,6 +72,11 @@ jobs: pushd $BUILD_DIR rm LICENSE || true + for i in 1 2 3; do + docker pull "$IMAGE" && break + echo "docker pull attempt $i failed, retrying..." + sleep 10 + done docker run --rm -e NO_VCS_VERSION=1 -v $(pwd):/workspace -w /workspace $IMAGE bash -c '\ for python_version in cp311 cp312 cp313; do \ /opt/python/${python_version}-${python_version}/bin/pip install --upgrade "setuptools>=80" build; \