From 8dc4d879837727edaa03affe4ecedd55a4dbadfa Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 23:34:48 -0700 Subject: [PATCH] ci(web): react-apps-check shards run on Blacksmith 16 vCPU, failing over to 32 The four webview test shards (and their base replay) were the largest Blacksmith spend: about 99k 32 vCPU minutes a day on 2026-10-29/30. The 16 vCPU shards of the same days took 14% longer at p50 for half the per-minute price. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-d750cf10922c785efa64 --- .github/workflows/ci-web.yml | 12 ++++++++---- tests/test_web_validation.py | 9 ++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-web.yml b/.github/workflows/ci-web.yml index 4b78731f8909..f12f67481d34 100644 --- a/.github/workflows/ci-web.yml +++ b/.github/workflows/ci-web.yml @@ -292,10 +292,14 @@ jobs: needs: web-subarea-scope if: ${{ inputs.web == 'true' && needs.web-subarea-scope.outputs.react_apps == 'true' }} # `bun test` grows memory across the suite and was SIGKILLed (exit 137) on the - # 4 vCPU runner (run 37666132324). The 32 vCPU size has the memory and runs - # 32 files and 8 dev-server files at once (run-webviews-tests.sh). Fork pull + # 4 vCPU runner (run 37666132324). 16 vCPU has the memory and runs 16 files + # and 4 dev-server files at once (run-webviews-tests.sh). These shards were the + # largest Blacksmith spend (about 99k 32 vCPU minutes a day, 2026-10-29/30); + # the 16 vCPU shards of the same days took 14% longer at p50 (1107 s against + # 972 s, pull requests) for half the per-minute price, the longest 2139 s, under + # the 40 min timeout. The overflow failover still goes to 32 vCPU. Fork pull # requests run the same suite on the hosted runner. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-32vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404' }} # The 10-minute siblings in this workflow, plus the browser engines and the # browser-lane suites they run. The full suite grew to 22 min in one process. # Deterministic file shards keep per-file isolation. Each shard warms one @@ -613,7 +617,7 @@ jobs: react-apps-base-check: needs: [web-subarea-scope, react-apps-check-shard] if: ${{ github.event_name == 'pull_request' && inputs.web == 'true' && needs.web-subarea-scope.outputs.react_apps == 'true' }} - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-32vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404' }} timeout-minutes: 40 name: react-apps-base-check (${{ matrix.shard }}) strategy: diff --git a/tests/test_web_validation.py b/tests/test_web_validation.py index 5c7a77be55b1..bdb807ab4312 100644 --- a/tests/test_web_validation.py +++ b/tests/test_web_validation.py @@ -214,13 +214,16 @@ def test_webviews_tests_run_on_a_runner_with_room_for_the_suite(self): self.assertIn("scripts/ci/run-webviews-tests.sh", block) self.assertNotIn("vars.LINUX_RUNNER", runs_on) self.assertNotIn("4vcpu", runs_on) - self.assertIn("'blacksmith-32vcpu-ubuntu-2404'", runs_on) + # 16 vCPU (memory for the suite at half the 32 vCPU price), failing over to 32. + self.assertTrue(runs_on.endswith("'blacksmith-16vcpu-ubuntu-2404' }}"), runs_on) + self.assertIn("vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-32vcpu-ubuntu-2404'", runs_on) def test_webviews_tests_shard_on_wide_runners(self): """Two 8 vCPU shards each ran about 65 dev-server files two at a time for 12 of their 14-16 minutes. Twelve 8 vCPU shards took 14.3 min: each paid a cold Vite warm-up. Four 32 vCPU shards run eight dev-server files at once on a shared warm - cache; Blacksmith runners are free and ran 256 jobs at 32 vCPU without queueing.""" + cache. Blacksmith is a capped paid overflow now (2026-10-30), so the four shards run at + 16 vCPU: 14% slower at p50 than 32 for half the price.""" workflow = (ROOT / ".github/workflows/ci-web.yml").read_text() start = workflow.index(" react-apps-check-shard:") match = re.search(r"\n [A-Za-z0-9_-]+:", workflow[start + 3 :]) @@ -228,7 +231,7 @@ def test_webviews_tests_shard_on_wide_runners(self): shards = [int(n) for n in re.findall(r"^ - shard: (\d+)$", block, re.MULTILINE)] counts = set(re.findall(r"^ shard_count: (\d+)$", block, re.MULTILINE)) self.assertEqual(len(shards), 4) - self.assertIn("'blacksmith-32vcpu-ubuntu-2404'", block) + self.assertIn("'blacksmith-16vcpu-ubuntu-2404' }}", block) self.assertEqual(shards, list(range(1, len(shards) + 1))) self.assertEqual(counts, {str(len(shards))}) for step in ("Generated bundle merge driver", "Check diff viewer eager JS budget",