From 84f01f92fc91e765f585ea46b688cee4296b115c Mon Sep 17 00:00:00 2001 From: "Nick L." Date: Tue, 28 Jul 2026 14:31:10 -0700 Subject: [PATCH] ci: triage chronic fork CI reds (zizmor, OSV, image scan, root-path tests) Four checks had been failing on essentially every push or schedule against litellm_internal_staging, so a genuine regression could no longer be told apart from the standing noise. zizmor exited 13 on two action pins whose version comment did not match the commit they pinned: both used the moving `v6` tag as a comment while pinning setup-node v6.4.0's SHA. Repinned to v6.5.0 with an exact comment. osv-scan reported gitpython 3.1.52 (5 advisories, up to 8.8) plus postcss 8.5.13 and brace-expansion 5.0.7 on the dashboard. All three have fixed releases, so they are bumped rather than ignored. gitpython arrives through the optional mlflow extra and is not in the shipped image; postcss and brace-expansion are build/dev only, but a fix exists so there is no reason to carry them. Grype's image-scan findings (pypdf, pyasn1) were already resolved by the upstream sync in #24; only the daily schedule, which ran against the pre-merge commit, was still red. test-server-root-path was not asserting anything. Since 2026-07-04 every run hung in `playwright install` immediately after the Chrome-for-Testing zip reached 100%, then died at the 30 minute job cap as "cancelled". The retry wrapper added in #32406 cannot recover from that because no attempt ever returns. Each browser-install attempt is now bounded by `timeout`, the apt half is split out so a SIGTERM cannot strand dpkg, and every long step got its own cap so a failure names the step instead of surfacing as a bare cancelled job. DEBUG=pw:install is set so the next occurrence says where it stopped. The npm cache key also pointed at the dashboard lockfile while `npm ci` runs in tests/e2e/ui, so it never hit; it now points at the lockfile actually installed. --- .github/workflows/check-ui-api-types.yml | 2 +- .github/workflows/test_server_root_path.yml | 33 ++++++++++++++++++--- ui/litellm-dashboard/package-lock.json | 18 +++++------ ui/litellm-dashboard/package.json | 6 ++-- uv.lock | 8 ++--- 5 files changed, 46 insertions(+), 21 deletions(-) diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml index 30d360cb5d80..c08da044003f 100644 --- a/.github/workflows/check-ui-api-types.yml +++ b/.github/workflows/check-ui-api-types.yml @@ -54,7 +54,7 @@ jobs: run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Set up Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: "24.16.0" cache: "npm" diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml index ee07a65a76a4..863ebf523b07 100644 --- a/.github/workflows/test_server_root_path.yml +++ b/.github/workflows/test_server_root_path.yml @@ -13,6 +13,9 @@ on: jobs: test-server-root-path: runs-on: ubuntu-latest + # Backstop only. Every step that can stall carries its own cap below, so a + # failure names the step that timed out instead of showing up as a bare + # "cancelled" job 30 minutes later. timeout-minutes: 30 strategy: @@ -36,6 +39,7 @@ jobs: uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Build Docker image + timeout-minutes: 12 uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0 with: context: . @@ -102,14 +106,29 @@ jobs: exit 1 - name: Setup Node for Playwright - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: "24.16.0" cache: "npm" - cache-dependency-path: ui/litellm-dashboard/package-lock.json - + # npm ci below runs in tests/e2e/ui, so key the cache off that + # lockfile; the dashboard lockfile it used to point at is a different + # dependency set and never produced a hit. + cache-dependency-path: tests/e2e/ui/package-lock.json + + # Browser install is split from apt and bounded per attempt. Since + # 2026-07-04 `playwright install` has hung indefinitely right after the + # Chrome-for-Testing zip reaches 100%: the bytes arrive (the CDN object is + # intact), then the post-download handoff never completes. The retry + # wrapper from #32406 cannot recover from a hang because no attempt ever + # returns, so the job burned its full 30m cap on every run. `timeout` + # bounds each attempt so a retry can take over, the step cap bounds the + # worst case, and DEBUG=pw:install makes the next failure say where it + # stopped instead of going silent. - name: Install e2e deps and Chromium working-directory: tests/e2e/ui + timeout-minutes: 8 + env: + DEBUG: pw:install run: | retry() { local attempt=1 @@ -130,10 +149,16 @@ jobs: npm config set fetch-retry-maxtimeout 120000 retry npm ci - retry npx playwright install --with-deps chromium + + # install-deps is the apt half; keep it outside `timeout` so a SIGTERM + # can never leave a half-applied dpkg state behind. + retry npx playwright install-deps chromium + # 4 attempts x 75s + backoff fits inside the 8m step cap above. + retry timeout 75 npx playwright install chromium - name: Run SERVER_ROOT_PATH redirect e2e working-directory: tests/e2e/ui + timeout-minutes: 5 env: SERVER_ROOT_PATH: ${{ matrix.root_path }} run: npx playwright test --config=serverRootPath.config.ts diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 5f6b4b889b19..c94be846f029 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -67,7 +67,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.13", + "postcss": "8.5.18", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -5529,16 +5529,16 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/braces": { @@ -11064,9 +11064,9 @@ } }, "node_modules/postcss": { - "version": "8.5.13", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.13.tgz", - "integrity": "sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==", + "version": "8.5.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.18.tgz", + "integrity": "sha512-xdB1oSLHbz1vRWgCDalrCqEFTWzFlhqFC5tIHLMOSUIjhm3XXQ1qrFy8S/ESr1JYRRXqM3c1QFiMZUJdUTqyMQ==", "funding": [ { "type": "opencollective", @@ -11083,7 +11083,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 63bcdfb6076a..1d9877cdf87d 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -79,7 +79,7 @@ "jsdom": "27.4.0", "knip": "5.83.1", "openapi-typescript": "7.13.0", - "postcss": "8.5.13", + "postcss": "8.5.18", "prettier": "3.2.5", "tailwindcss": "4.3.2", "tw-animate-css": "1.4.0", @@ -90,13 +90,13 @@ "overrides": { "prismjs": "1.30.0", "js-yaml": "4.3.0", - "brace-expansion": "5.0.7", + "brace-expansion": "5.0.8", "glob": "13.0.0", "minimatch": "10.2.4", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", - "postcss": "8.5.13", + "postcss": "8.5.18", "esbuild": "0.28.1", "date-fns": "^4.4.0", "sharp": "^0.35.0" diff --git a/uv.lock b/uv.lock index 9c60ca1ee480..f448154854be 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-19T00:00:06.091071Z" +exclude-newer = "2026-07-25T21:28:04.689377Z" exclude-newer-span = "P3D" [manifest] @@ -2378,14 +2378,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.52" +version = "3.1.56" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/fd/df0bafa4eb5ea2f51e1adee9f7a94c8e62c5d180e65117045dfca3439c8a/gitpython-3.1.52.tar.gz", hash = "sha256:de0a8ad86274c6e75ae8b37dd055ba68f19818c813108642263227b20775b48e", size = 223726, upload-time = "2026-07-16T03:15:59.599Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/45/d45f94fa38b199862959cd7b5461a31f03746d75ef59363339fc0c394345/gitpython-3.1.56.tar.gz", hash = "sha256:127adf5c73f1a822e368301c4d5ffa5d305ce611eccab76f3336f9380a78ad0b", size = 229619, upload-time = "2026-07-25T07:41:43.179Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/90/04dff7c1e176bb1c3011ef1647393d368790da710d8dde1cdcfad301f45a/gitpython-3.1.52-py3-none-any.whl", hash = "sha256:79a36ee1f83523214a3f72d56cf1c4e490d577dc61af77e43dfe5862bd9da01a", size = 215366, upload-time = "2026-07-16T03:15:58.239Z" }, + { url = "https://files.pythonhosted.org/packages/06/27/8b9b039c7f026d4af8954dbbdcc9f101f8d6901daa3a65b065d8f450c0bd/gitpython-3.1.56-py3-none-any.whl", hash = "sha256:bedffdc4bdd2e3cf21b328711a552b0529751f08bff6591339d18492291ad035", size = 216644, upload-time = "2026-07-25T07:41:41.737Z" }, ] [[package]]