diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1eec324cd4..24ff45da4e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -263,18 +263,25 @@ jobs: echo "::warning::No Ubuntu archive apt source detected; leaving apt sources untouched" fi - name: Install Chromium - timeout-minutes: 15 + timeout-minutes: 20 run: | readonly install_attempts=2 - readonly install_timeout_minutes=5 + readonly install_timeout_minutes=7 readonly install_kill_grace_seconds=15 readonly apt_lock_attempts=12 readonly apt_lock_sleep_seconds=5 readonly retry_backoff_seconds=20 + readonly apt_lock_files="/var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock" + + # Playwright shells out to apt-get itself, so a lock held by the + # runner image's own apt activity used to surface inside the install + # as an instant "Could not get lock" failure. A global lock timeout + # makes every apt invocation wait for the holder instead. + echo 'DPkg::Lock::Timeout "300";' | sudo tee /etc/apt/apt.conf.d/99vf-lock-timeout >/dev/null wait_for_apt_locks() { for _ in $(seq 1 "${apt_lock_attempts}"); do - if ! sudo fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock >/dev/null 2>&1; then + if ! sudo fuser ${apt_lock_files} >/dev/null 2>&1; then return 0 fi sleep "${apt_lock_sleep_seconds}" @@ -282,10 +289,22 @@ jobs: return 1 } + # `timeout` signals only the wrapper process, so a timed-out attempt + # orphans the sudo apt-get underneath it. That orphan keeps holding + # the dpkg locks and starved every later attempt on this VM. Reap + # the lock holders and repair any half-configured packages before + # trying again. + reap_apt_lock_holders() { + sudo fuser -k -TERM ${apt_lock_files} >/dev/null 2>&1 || true + sleep 5 + sudo fuser -k -KILL ${apt_lock_files} >/dev/null 2>&1 || true + sudo dpkg --configure -a >/dev/null 2>&1 || true + } + for attempt in $(seq 1 "${install_attempts}"); do if ! wait_for_apt_locks; then - echo "::error::Timed out waiting for apt/dpkg locks" - exit 1 + echo "::warning::apt/dpkg locks still held; reaping the holders" + reap_apt_lock_holders fi timeout --signal=TERM --kill-after="${install_kill_grace_seconds}s" "${install_timeout_minutes}m" \ deno run -A npm:playwright install --with-deps chromium && exit 0 @@ -295,6 +314,7 @@ jobs: exit "$status" fi + reap_apt_lock_holders sleep "${retry_backoff_seconds}" done - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 @@ -355,18 +375,25 @@ jobs: echo "::warning::No Ubuntu archive apt source detected; leaving apt sources untouched" fi - name: Install Chromium - timeout-minutes: 15 + timeout-minutes: 20 run: | readonly install_attempts=2 - readonly install_timeout_minutes=5 + readonly install_timeout_minutes=7 readonly install_kill_grace_seconds=15 readonly apt_lock_attempts=12 readonly apt_lock_sleep_seconds=5 readonly retry_backoff_seconds=20 + readonly apt_lock_files="/var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock" + + # Playwright shells out to apt-get itself, so a lock held by the + # runner image's own apt activity used to surface inside the install + # as an instant "Could not get lock" failure. A global lock timeout + # makes every apt invocation wait for the holder instead. + echo 'DPkg::Lock::Timeout "300";' | sudo tee /etc/apt/apt.conf.d/99vf-lock-timeout >/dev/null wait_for_apt_locks() { for _ in $(seq 1 "${apt_lock_attempts}"); do - if ! sudo fuser /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock >/dev/null 2>&1; then + if ! sudo fuser ${apt_lock_files} >/dev/null 2>&1; then return 0 fi sleep "${apt_lock_sleep_seconds}" @@ -374,10 +401,22 @@ jobs: return 1 } + # `timeout` signals only the wrapper process, so a timed-out attempt + # orphans the sudo apt-get underneath it. That orphan keeps holding + # the dpkg locks and starved every later attempt on this VM. Reap + # the lock holders and repair any half-configured packages before + # trying again. + reap_apt_lock_holders() { + sudo fuser -k -TERM ${apt_lock_files} >/dev/null 2>&1 || true + sleep 5 + sudo fuser -k -KILL ${apt_lock_files} >/dev/null 2>&1 || true + sudo dpkg --configure -a >/dev/null 2>&1 || true + } + for attempt in $(seq 1 "${install_attempts}"); do if ! wait_for_apt_locks; then - echo "::error::Timed out waiting for apt/dpkg locks" - exit 1 + echo "::warning::apt/dpkg locks still held; reaping the holders" + reap_apt_lock_holders fi timeout --signal=TERM --kill-after="${install_kill_grace_seconds}s" "${install_timeout_minutes}m" \ deno run -A npm:playwright install --with-deps chromium && exit 0 @@ -387,6 +426,7 @@ jobs: exit "$status" fi + reap_apt_lock_holders sleep "${retry_backoff_seconds}" done - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 diff --git a/scripts/ci/setup-deno-workflow.test.ts b/scripts/ci/setup-deno-workflow.test.ts index 6e7e8f5819..3515579d81 100644 --- a/scripts/ci/setup-deno-workflow.test.ts +++ b/scripts/ci/setup-deno-workflow.test.ts @@ -18,7 +18,7 @@ const CACHE_SAVE_ACTION = const MAX_SETUP_MINUTES = 5; const MAX_CACHE_SETUP_MINUTES = 10; const CACHE_PRODUCER_JOB = "tests"; -const CHROMIUM_STEP_MINUTES = 15; +const CHROMIUM_STEP_MINUTES = 20; const CHROMIUM_OVERHEAD_MARGIN_SECONDS = 120; /** * How long an upstream outage on the two REQUIRED Deno downloads must be