From e2cd2d4ab82a59f4b915c5fd68f3fda53566bc6b Mon Sep 17 00:00:00 2001 From: James Wiesebron Date: Tue, 12 May 2026 20:36:39 -0700 Subject: [PATCH 1/2] sandbox: retry deadsnakes apt install on transient PPA failures The deadsnakes PPA occasionally returns 5xx during `apt-get update`, leaving python3.14-venv/dev unlocatable and failing `make build` (see PR #2694 Integration Tests run). Wrap the post-PPA `update + install` in both the `repo-deps` and `base` sandbox stages with a 5-attempt retry loop and pass `Acquire::Retries=3` for in-call resilience. --- sandbox/Dockerfile | 63 +++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile index aa52403ce4..8633b70c09 100644 --- a/sandbox/Dockerfile +++ b/sandbox/Dockerfile @@ -28,14 +28,28 @@ ENV GOMODCACHE=/usr/local/pkg/mod # pyproject.toml. Without it, uv downloads a managed Python into /root/ which # creates broken symlinks when the .venv is persisted and restored for the # non-root runtime user. -RUN apt-get update && apt-get install -y \ - python3 python3-pip \ - git curl wget make build-essential pkg-config \ - # Common build deps needed by repo build_commands - jq unzip software-properties-common \ +RUN set -eux \ + && apt-get update && apt-get install -y \ + python3 python3-pip \ + git curl wget make build-essential pkg-config \ + jq unzip software-properties-common \ && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get update \ - && apt-get install -y python3.14 python3.14-venv python3.14-dev \ + # deadsnakes PPA occasionally returns 5xx during apt-get update, leaving + # python3.14* packages unlocatable on install (see PR #2694 CI failure). + # Retry the post-PPA update+install with backoff; Acquire::Retries handles + # short blips inside a single apt invocation, the outer loop handles + # longer outages. + && for i in 1 2 3 4 5; do \ + if apt-get update -o Acquire::Retries=3 \ + && apt-get install -y python3.14 python3.14-venv python3.14-dev; then \ + break; \ + fi; \ + if [ "$i" = 5 ]; then \ + echo "deadsnakes install failed after 5 attempts" >&2; exit 1; \ + fi; \ + echo "deadsnakes install attempt $i failed, sleeping $((i*15))s..." >&2; \ + sleep $((i*15)); \ + done \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.14 1 \ && update-alternatives --set python3 /usr/bin/python3.14 \ && python3.14 -m ensurepip --upgrade \ @@ -121,17 +135,30 @@ RUN curl -fsSL https://repo.charm.sh/apt/gpg.key | gpg --dearmor -o /usr/share/k # to pick up deadsnakes packages. # python3-pip from apt targets the system Python (3.10), not 3.14 — so we # omit it and bootstrap pip via ensurepip after setting update-alternatives. -RUN apt-get update && \ - add-apt-repository -y ppa:deadsnakes/ppa && \ - apt-get update && \ - apt-get install -y python3.14 python3.14-venv python3.14-dev && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.14 1 && \ - update-alternatives --set python3 /usr/bin/python3.14 && \ - # Ensure 'python' command maps to python3 (many tools expect this) - update-alternatives --install /usr/bin/python python /usr/bin/python3.14 1 && \ - # Bootstrap pip for Python 3.14 (ensurepip installs pip3 to /usr/local/bin) - python3.14 -m ensurepip --upgrade && \ - python3.14 -m pip install --no-cache-dir --upgrade pip +RUN set -eux \ + && apt-get update \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + # deadsnakes PPA occasionally returns 5xx during apt-get update, leaving + # python3.14* packages unlocatable on install (see PR #2694 CI failure). + # Retry the post-PPA update+install with backoff; Acquire::Retries handles + # short blips inside a single apt invocation, the outer loop handles + # longer outages. + && for i in 1 2 3 4 5; do \ + if apt-get update -o Acquire::Retries=3 \ + && apt-get install -y python3.14 python3.14-venv python3.14-dev; then \ + break; \ + fi; \ + if [ "$i" = 5 ]; then \ + echo "deadsnakes install failed after 5 attempts" >&2; exit 1; \ + fi; \ + echo "deadsnakes install attempt $i failed, sleeping $((i*15))s..." >&2; \ + sleep $((i*15)); \ + done \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.14 1 \ + && update-alternatives --set python3 /usr/bin/python3.14 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.14 1 \ + && python3.14 -m ensurepip --upgrade \ + && python3.14 -m pip install --no-cache-dir --upgrade pip # Install pyyaml early - required by docker-setup.py for reading config RUN pip3 install --no-cache-dir pyyaml From c0e3442ff22f60e135c5f429c34f8f7fb90710f5 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 03:52:42 +0000 Subject: [PATCH 2/2] sandbox: trip retry on apt update index errors Add -o APT::Update::Error-Mode=any to the apt-get update calls inside both deadsnakes retry loops. This promotes per-source index errors to a non-zero exit, so a deadsnakes 5xx fails update directly instead of silently leaving an empty index and surfacing later as an unlocatable install. The outer retry loop trips on the cleaner signal and skips the doomed install attempt. Per review feedback on #2695. --- sandbox/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sandbox/Dockerfile b/sandbox/Dockerfile index 8633b70c09..91cf868f06 100644 --- a/sandbox/Dockerfile +++ b/sandbox/Dockerfile @@ -38,9 +38,11 @@ RUN set -eux \ # python3.14* packages unlocatable on install (see PR #2694 CI failure). # Retry the post-PPA update+install with backoff; Acquire::Retries handles # short blips inside a single apt invocation, the outer loop handles - # longer outages. + # longer outages. APT::Update::Error-Mode=any promotes per-source index + # errors to a non-zero exit, so a deadsnakes 5xx trips the retry directly + # instead of leaking through as an unlocatable-package install failure. && for i in 1 2 3 4 5; do \ - if apt-get update -o Acquire::Retries=3 \ + if apt-get update -o Acquire::Retries=3 -o APT::Update::Error-Mode=any \ && apt-get install -y python3.14 python3.14-venv python3.14-dev; then \ break; \ fi; \ @@ -142,9 +144,11 @@ RUN set -eux \ # python3.14* packages unlocatable on install (see PR #2694 CI failure). # Retry the post-PPA update+install with backoff; Acquire::Retries handles # short blips inside a single apt invocation, the outer loop handles - # longer outages. + # longer outages. APT::Update::Error-Mode=any promotes per-source index + # errors to a non-zero exit, so a deadsnakes 5xx trips the retry directly + # instead of leaking through as an unlocatable-package install failure. && for i in 1 2 3 4 5; do \ - if apt-get update -o Acquire::Retries=3 \ + if apt-get update -o Acquire::Retries=3 -o APT::Update::Error-Mode=any \ && apt-get install -y python3.14 python3.14-venv python3.14-dev; then \ break; \ fi; \