From 51748829194f311f7b51332b2fa2af648d556c50 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 08:33:33 +0000 Subject: [PATCH 1/7] Build: Speed up Windows jobs on CircleCI by optimizing cache management --- .circleci/config.yml | 190 +++++++++++++++++- .circleci/src/jobs/bench-packages.yml | 6 + .circleci/src/jobs/bench-sandboxes.yml | 6 + .circleci/src/jobs/build.yml | 34 ++-- .../src/jobs/chromatic-internal-storybook.yml | 6 + .circleci/src/jobs/chromatic-sandboxes.yml | 15 ++ .circleci/src/jobs/create-sandboxes.yml | 18 +- .circleci/src/jobs/e2e-dev.yml | 15 ++ .circleci/src/jobs/e2e-production.yml | 15 ++ .circleci/src/jobs/e2e-ui-vitest-3.yml | 6 + .circleci/src/jobs/e2e-ui.yml | 6 + .circleci/src/jobs/knip.yml | 6 + .circleci/src/jobs/lint.yml | 6 + .circleci/src/jobs/script-checks.yml | 6 + .circleci/src/jobs/stories-tests.yml | 6 + .circleci/src/jobs/test-init-empty.yml | 6 + .circleci/src/jobs/test-init-features.yml | 6 + .circleci/src/jobs/test-portable-stories.yml | 6 + .circleci/src/jobs/test-runner-production.yml | 15 ++ .circleci/src/jobs/test-yarn-pnp.yml | 6 + .circleci/src/jobs/unit-tests.yml | 6 + .circleci/src/jobs/vitest-integration.yml | 15 ++ 22 files changed, 373 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2cc4eda78dc1..7caed1acd44d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,6 +116,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - when: condition: and: @@ -177,6 +183,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task bench) @@ -213,18 +225,17 @@ jobs: yarn local-registry --publish name: Publish to Verdaccio - report-workflow-on-failure - - store_artifacts: - path: code/bench/esbuild-metafiles - save_cache: - key: build-yarn-2-cache-v5--{{ checksum "code/yarn.lock" }}--{{ checksum "scripts/yarn.lock" }} - name: Save Yarn cache + key: code-node-modules-{{ checksum "code/yarn.lock" }} paths: - - ~/.yarn/berry/cache + - code/node_modules + - save_cache: + key: scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + paths: + - scripts/node_modules - persist_to_workspace: paths: - - code/node_modules - code/addons - - scripts/node_modules - code/bench - code/examples - code/frameworks @@ -298,6 +309,12 @@ jobs: - checkout - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: yarn storybook:ui:chromatic name: Running Chromatic @@ -317,6 +334,21 @@ jobs: - checkout - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task chromatic) @@ -353,6 +385,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | # Enable corepack @@ -370,6 +408,7 @@ jobs: if [[ $TEMPLATE != bench/* ]]; then yarn --cwd scripts jiti ./event-log-checker.ts build $TEMPLATE fi + # Clean node_modules before caching sandbox to keep cache light cd sandbox/$(yarn get-sandbox-dir --template $TEMPLATE) && rm -rf node_modules environment: STORYBOOK_TELEMETRY_DEBUG: 1 @@ -377,10 +416,17 @@ jobs: name: Create Sandboxes - report-workflow-on-failure: template: $(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - - persist_to_workspace: + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - save_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} paths: - sandbox/** - root: . - store_test_results: path: test-results e2e-dev: @@ -396,6 +442,21 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests-dev) @@ -426,6 +487,21 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests) @@ -452,6 +528,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: yarn install --no-immutable environment: @@ -477,6 +559,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: yarn install --no-immutable environment: @@ -502,6 +590,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd code @@ -518,6 +612,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd code @@ -558,6 +658,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd scripts @@ -607,6 +713,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd code @@ -631,6 +743,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - when: condition: equal: @@ -932,6 +1050,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: background: true command: | @@ -969,6 +1093,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: yarn install --no-immutable environment: @@ -1025,6 +1155,21 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner) @@ -1053,6 +1198,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: yarn install --no-immutable environment: @@ -1074,6 +1225,12 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd code @@ -1097,6 +1254,21 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task vitest-integration) diff --git a/.circleci/src/jobs/bench-packages.yml b/.circleci/src/jobs/bench-packages.yml index 599407df051a..30be607eaea0 100644 --- a/.circleci/src/jobs/bench-packages.yml +++ b/.circleci/src/jobs/bench-packages.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} # if there is a base branch AND a PR number in parameters, benchmark packages against those # this happens when run against a PR - when: diff --git a/.circleci/src/jobs/bench-sandboxes.yml b/.circleci/src/jobs/bench-sandboxes.yml index a5e74639f757..3c521b97da54 100644 --- a/.circleci/src/jobs/bench-sandboxes.yml +++ b/.circleci/src/jobs/bench-sandboxes.yml @@ -13,6 +13,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/build.yml b/.circleci/src/jobs/build.yml index c578a44194d8..36898706d88b 100644 --- a/.circleci/src/jobs/build.yml +++ b/.circleci/src/jobs/build.yml @@ -21,25 +21,25 @@ steps: cd code yarn local-registry --publish - report-workflow-on-failure - - store_artifacts: - path: code/bench/esbuild-metafiles - save_cache: - name: Save Yarn cache - key: build-yarn-2-cache-v5--{{ checksum "code/yarn.lock" }}--{{ checksum "scripts/yarn.lock" }} + key: code-node-modules-{{ checksum "code/yarn.lock" }} paths: - - ~/.yarn/berry/cache + - code/node_modules + - save_cache: + key: scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + paths: + - scripts/node_modules + # Persist source/artifacts only (exclude node_modules) - persist_to_workspace: root: . paths: - - code/node_modules - - code/addons - - scripts/node_modules - - code/bench - - code/examples - - code/frameworks - - code/lib - - code/core - - code/builders - - code/renderers - - code/presets - - .verdaccio-cache + - code/addons + - code/bench + - code/examples + - code/frameworks + - code/lib + - code/core + - code/builders + - code/renderers + - code/presets + - .verdaccio-cache diff --git a/.circleci/src/jobs/chromatic-internal-storybook.yml b/.circleci/src/jobs/chromatic-internal-storybook.yml index 880755fc1437..b63efdce26f8 100644 --- a/.circleci/src/jobs/chromatic-internal-storybook.yml +++ b/.circleci/src/jobs/chromatic-internal-storybook.yml @@ -10,6 +10,12 @@ steps: - checkout - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Running Chromatic command: yarn storybook:ui:chromatic diff --git a/.circleci/src/jobs/chromatic-sandboxes.yml b/.circleci/src/jobs/chromatic-sandboxes.yml index 24ec537f67f6..51d86ca5acad 100644 --- a/.circleci/src/jobs/chromatic-sandboxes.yml +++ b/.circleci/src/jobs/chromatic-sandboxes.yml @@ -12,6 +12,21 @@ steps: - checkout - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/create-sandboxes.yml b/.circleci/src/jobs/create-sandboxes.yml index 0fc844390ea5..1f4055f2c703 100644 --- a/.circleci/src/jobs/create-sandboxes.yml +++ b/.circleci/src/jobs/create-sandboxes.yml @@ -13,6 +13,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Setup Corepack command: | @@ -31,14 +37,22 @@ steps: if [[ $TEMPLATE != bench/* ]]; then yarn --cwd scripts jiti ./event-log-checker.ts build $TEMPLATE fi + # Clean node_modules before caching sandbox to keep cache light cd sandbox/$(yarn get-sandbox-dir --template $TEMPLATE) && rm -rf node_modules environment: STORYBOOK_TELEMETRY_DEBUG: 1 STORYBOOK_TELEMETRY_URL: 'http://localhost:6007/event-log' - report-workflow-on-failure: template: $(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - - persist_to_workspace: - root: . + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - save_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} paths: - sandbox/** - store_test_results: diff --git a/.circleci/src/jobs/e2e-dev.yml b/.circleci/src/jobs/e2e-dev.yml index f404104c9ce4..f582af174159 100644 --- a/.circleci/src/jobs/e2e-dev.yml +++ b/.circleci/src/jobs/e2e-dev.yml @@ -13,6 +13,21 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/e2e-production.yml b/.circleci/src/jobs/e2e-production.yml index 61f9c12d5030..0f6c4acca97b 100644 --- a/.circleci/src/jobs/e2e-production.yml +++ b/.circleci/src/jobs/e2e-production.yml @@ -13,6 +13,21 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/e2e-ui-vitest-3.yml b/.circleci/src/jobs/e2e-ui-vitest-3.yml index 79aded7faf0f..76b48e2a22d5 100644 --- a/.circleci/src/jobs/e2e-ui-vitest-3.yml +++ b/.circleci/src/jobs/e2e-ui-vitest-3.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Install dependencies command: yarn install --no-immutable diff --git a/.circleci/src/jobs/e2e-ui.yml b/.circleci/src/jobs/e2e-ui.yml index 9577b3535044..513b2974ae5f 100644 --- a/.circleci/src/jobs/e2e-ui.yml +++ b/.circleci/src/jobs/e2e-ui.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Install dependencies command: yarn install --no-immutable diff --git a/.circleci/src/jobs/knip.yml b/.circleci/src/jobs/knip.yml index 8a8db8013529..8d001eebe9d1 100644 --- a/.circleci/src/jobs/knip.yml +++ b/.circleci/src/jobs/knip.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Knip command: | diff --git a/.circleci/src/jobs/lint.yml b/.circleci/src/jobs/lint.yml index 66a22f3a348f..95dbc2db5e96 100644 --- a/.circleci/src/jobs/lint.yml +++ b/.circleci/src/jobs/lint.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Lint command: | diff --git a/.circleci/src/jobs/script-checks.yml b/.circleci/src/jobs/script-checks.yml index 42389005f5e8..e4b65f1c1021 100644 --- a/.circleci/src/jobs/script-checks.yml +++ b/.circleci/src/jobs/script-checks.yml @@ -5,6 +5,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Check parallelism count command: | diff --git a/.circleci/src/jobs/stories-tests.yml b/.circleci/src/jobs/stories-tests.yml index 588000bb34ae..abc9b6e7c77c 100644 --- a/.circleci/src/jobs/stories-tests.yml +++ b/.circleci/src/jobs/stories-tests.yml @@ -7,6 +7,12 @@ steps: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | cd code diff --git a/.circleci/src/jobs/test-init-empty.yml b/.circleci/src/jobs/test-init-empty.yml index e0fb76994bc9..211fcfdbe313 100644 --- a/.circleci/src/jobs/test-init-empty.yml +++ b/.circleci/src/jobs/test-init-empty.yml @@ -13,6 +13,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - when: condition: equal: ['npm', << parameters.packageManager >>] diff --git a/.circleci/src/jobs/test-init-features.yml b/.circleci/src/jobs/test-init-features.yml index d590ee7c78cf..1bab8da54a03 100644 --- a/.circleci/src/jobs/test-init-features.yml +++ b/.circleci/src/jobs/test-init-features.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Verdaccio background: true diff --git a/.circleci/src/jobs/test-portable-stories.yml b/.circleci/src/jobs/test-portable-stories.yml index d2b26147045d..b87c34c831c6 100644 --- a/.circleci/src/jobs/test-portable-stories.yml +++ b/.circleci/src/jobs/test-portable-stories.yml @@ -11,6 +11,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Install dependencies command: yarn install --no-immutable diff --git a/.circleci/src/jobs/test-runner-production.yml b/.circleci/src/jobs/test-runner-production.yml index 02d32ba41766..e33ea1a7fdbd 100644 --- a/.circleci/src/jobs/test-runner-production.yml +++ b/.circleci/src/jobs/test-runner-production.yml @@ -13,6 +13,21 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/test-yarn-pnp.yml b/.circleci/src/jobs/test-yarn-pnp.yml index b89e4d8bd540..57db45c15338 100644 --- a/.circleci/src/jobs/test-yarn-pnp.yml +++ b/.circleci/src/jobs/test-yarn-pnp.yml @@ -7,6 +7,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Install dependencies command: yarn install --no-immutable diff --git a/.circleci/src/jobs/unit-tests.yml b/.circleci/src/jobs/unit-tests.yml index c94cbd725bbc..f20b775fc5fa 100644 --- a/.circleci/src/jobs/unit-tests.yml +++ b/.circleci/src/jobs/unit-tests.yml @@ -9,6 +9,12 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: name: Run tests # TEST_FILES should include all relevant test files, including story files, but not e2e-tests diff --git a/.circleci/src/jobs/vitest-integration.yml b/.circleci/src/jobs/vitest-integration.yml index 1d52ff4c1896..bee71b28b587 100644 --- a/.circleci/src/jobs/vitest-integration.yml +++ b/.circleci/src/jobs/vitest-integration.yml @@ -13,6 +13,21 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) + echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + source $BASH_ENV + echo "Computed sandbox template: $sandbox-template" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} - run: name: Install sandbox dependencies command: | From 73894823a77660b5e3980ec3081f4842ea7176c9 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 09:47:29 +0000 Subject: [PATCH 2/7] Fix get-template parallelism validation --- scripts/get-template.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/get-template.ts b/scripts/get-template.ts index f16fa4994668..df2a503998e0 100644 --- a/scripts/get-template.ts +++ b/scripts/get-template.ts @@ -50,7 +50,8 @@ export async function getTemplate( { index, total }: { index: number; total: number } ) { let potentialTemplateKeys: TemplateKey[] = []; - if (await pathExists(sandboxDir)) { + const isLocalSandbox = await pathExists(sandboxDir); + if (isLocalSandbox) { const sandboxes = await getDirectories(sandboxDir); potentialTemplateKeys = sandboxes .map((dirName) => { @@ -73,7 +74,7 @@ export async function getTemplate( return isTaskSkipped(currentTemplate, scriptName); }); - if (potentialTemplateKeys.length !== total) { + if (!isLocalSandbox && potentialTemplateKeys.length !== total) { throw new Error(dedent`Circle parallelism set incorrectly. Parallelism is set to ${total}, but there are ${ @@ -332,9 +333,9 @@ async function run({ cadence, task, check, fix }: RunOptions) { } const { CIRCLE_NODE_INDEX = 0, CIRCLE_NODE_TOTAL = 1 } = process.env; - console.log( await getTemplate(cadence as Cadence, task, { + // Convert to integer index: +CIRCLE_NODE_INDEX, total: +CIRCLE_NODE_TOTAL, }) From a4c64a9ec00c3712b4065951200cd35ea2b6267b Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 09:54:42 +0000 Subject: [PATCH 3/7] Fix cache key checksum --- .circleci/src/jobs/create-sandboxes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/src/jobs/create-sandboxes.yml b/.circleci/src/jobs/create-sandboxes.yml index 1f4055f2c703..e447f7792271 100644 --- a/.circleci/src/jobs/create-sandboxes.yml +++ b/.circleci/src/jobs/create-sandboxes.yml @@ -48,11 +48,11 @@ steps: name: Compute sandbox cache key command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + echo "Computed sandbox template: $TEMPLATE" - save_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} paths: - sandbox/** - store_test_results: From 353958fb2e295aff82c5a83cfa317a19569412f8 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 09:57:53 +0000 Subject: [PATCH 4/7] Fix cache key for sandbox folders --- .circleci/config.yml | 82 ++++++++++++------- .circleci/src/jobs/check-sandboxes.yml | 14 ++++ .circleci/src/jobs/chromatic-sandboxes.yml | 9 +- .circleci/src/jobs/create-sandboxes.yml | 3 +- .circleci/src/jobs/e2e-dev.yml | 9 +- .circleci/src/jobs/e2e-production.yml | 9 +- .circleci/src/jobs/test-runner-dev.yml | 14 ++++ .circleci/src/jobs/test-runner-production.yml | 9 +- .circleci/src/jobs/vitest-integration.yml | 9 +- 9 files changed, 101 insertions(+), 57 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7caed1acd44d..55784f2baa82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -287,6 +287,20 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task check-sandbox) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task check-sandbox) @@ -342,13 +356,12 @@ jobs: - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task chromatic) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task chromatic) @@ -419,14 +432,13 @@ jobs: - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - save_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} paths: - - sandbox/** + - sandbox - store_test_results: path: test-results e2e-dev: @@ -450,13 +462,12 @@ jobs: - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests-dev) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests-dev) @@ -495,13 +506,12 @@ jobs: - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests) @@ -1135,6 +1145,20 @@ jobs: clone_options: --depth 1 --verbose - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" + name: Compute sandbox cache key + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: yarn task --task test-runner-dev --template $(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) --no-link --start-from=never --junit name: Running Test Runner in Dev mode @@ -1163,13 +1187,12 @@ jobs: - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner) @@ -1262,13 +1285,12 @@ jobs: - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} - run: command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task vitest-integration) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task vitest-integration) diff --git a/.circleci/src/jobs/check-sandboxes.yml b/.circleci/src/jobs/check-sandboxes.yml index bbc893b9fad1..c56903795b07 100644 --- a/.circleci/src/jobs/check-sandboxes.yml +++ b/.circleci/src/jobs/check-sandboxes.yml @@ -13,6 +13,20 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task check-sandbox) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/chromatic-sandboxes.yml b/.circleci/src/jobs/chromatic-sandboxes.yml index 51d86ca5acad..36d86d63ce9c 100644 --- a/.circleci/src/jobs/chromatic-sandboxes.yml +++ b/.circleci/src/jobs/chromatic-sandboxes.yml @@ -21,12 +21,11 @@ steps: - run: name: Compute sandbox cache key command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task chromatic) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/create-sandboxes.yml b/.circleci/src/jobs/create-sandboxes.yml index e447f7792271..3518ef5c071c 100644 --- a/.circleci/src/jobs/create-sandboxes.yml +++ b/.circleci/src/jobs/create-sandboxes.yml @@ -49,11 +49,10 @@ steps: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt - source $BASH_ENV echo "Computed sandbox template: $TEMPLATE" - save_cache: key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} paths: - - sandbox/** + - sandbox - store_test_results: path: test-results diff --git a/.circleci/src/jobs/e2e-dev.yml b/.circleci/src/jobs/e2e-dev.yml index f582af174159..ab03d09cb266 100644 --- a/.circleci/src/jobs/e2e-dev.yml +++ b/.circleci/src/jobs/e2e-dev.yml @@ -22,12 +22,11 @@ steps: - run: name: Compute sandbox cache key command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests-dev) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/e2e-production.yml b/.circleci/src/jobs/e2e-production.yml index 0f6c4acca97b..db9731ca1f28 100644 --- a/.circleci/src/jobs/e2e-production.yml +++ b/.circleci/src/jobs/e2e-production.yml @@ -22,12 +22,11 @@ steps: - run: name: Compute sandbox cache key command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/test-runner-dev.yml b/.circleci/src/jobs/test-runner-dev.yml index 36f7db608bb1..3842b9d38041 100644 --- a/.circleci/src/jobs/test-runner-dev.yml +++ b/.circleci/src/jobs/test-runner-dev.yml @@ -13,6 +13,20 @@ steps: clone_options: '--depth 1 --verbose' - attach_workspace: at: . + - restore_cache: + keys: + - code-node-modules-{{ checksum "code/yarn.lock" }} + - restore_cache: + keys: + - scripts-node-modules-{{ checksum "scripts/yarn.lock" }} + - run: + name: Compute sandbox cache key + command: | + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" + - restore_cache: + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Running Test Runner in Dev mode command: yarn task --task test-runner-dev --template $(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) --no-link --start-from=never --junit diff --git a/.circleci/src/jobs/test-runner-production.yml b/.circleci/src/jobs/test-runner-production.yml index e33ea1a7fdbd..c2a29402f13f 100644 --- a/.circleci/src/jobs/test-runner-production.yml +++ b/.circleci/src/jobs/test-runner-production.yml @@ -22,12 +22,11 @@ steps: - run: name: Compute sandbox cache key command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/vitest-integration.yml b/.circleci/src/jobs/vitest-integration.yml index bee71b28b587..b3e43ce1101c 100644 --- a/.circleci/src/jobs/vitest-integration.yml +++ b/.circleci/src/jobs/vitest-integration.yml @@ -22,12 +22,11 @@ steps: - run: name: Compute sandbox cache key command: | - TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task sandbox) - echo "sandbox-template=$TEMPLATE" >> $BASH_ENV - source $BASH_ENV - echo "Computed sandbox template: $sandbox-template" + TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task vitest-integration) + echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt + echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum "sandbox/${sandbox-template}/package.json" }} + key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | From 5f08899389e767178270771256d3103fa409cb6b Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 10:23:42 +0000 Subject: [PATCH 5/7] Fix cache key to use revision instead of build number for sandbox jobs --- .circleci/config.yml | 16 ++++++++-------- .circleci/src/jobs/check-sandboxes.yml | 2 +- .circleci/src/jobs/chromatic-sandboxes.yml | 2 +- .circleci/src/jobs/create-sandboxes.yml | 2 +- .circleci/src/jobs/e2e-dev.yml | 2 +- .circleci/src/jobs/e2e-production.yml | 2 +- .circleci/src/jobs/test-runner-dev.yml | 2 +- .circleci/src/jobs/test-runner-production.yml | 2 +- .circleci/src/jobs/vitest-integration.yml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55784f2baa82..0de3764732a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -300,7 +300,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task check-sandbox) @@ -361,7 +361,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task chromatic) @@ -436,7 +436,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - save_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} paths: - sandbox - store_test_results: @@ -467,7 +467,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests-dev) @@ -511,7 +511,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task e2e-tests) @@ -1158,7 +1158,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: yarn task --task test-runner-dev --template $(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) --no-link --start-from=never --junit name: Running Test Runner in Dev mode @@ -1192,7 +1192,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner) @@ -1290,7 +1290,7 @@ jobs: echo "Computed sandbox template: $TEMPLATE" name: Compute sandbox cache key - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: command: | TEMPLATE=$(yarn get-template --cadence << pipeline.parameters.workflow >> --task vitest-integration) diff --git a/.circleci/src/jobs/check-sandboxes.yml b/.circleci/src/jobs/check-sandboxes.yml index c56903795b07..a1468af60985 100644 --- a/.circleci/src/jobs/check-sandboxes.yml +++ b/.circleci/src/jobs/check-sandboxes.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/chromatic-sandboxes.yml b/.circleci/src/jobs/chromatic-sandboxes.yml index 36d86d63ce9c..8cad4a2fc69e 100644 --- a/.circleci/src/jobs/chromatic-sandboxes.yml +++ b/.circleci/src/jobs/chromatic-sandboxes.yml @@ -25,7 +25,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/create-sandboxes.yml b/.circleci/src/jobs/create-sandboxes.yml index 3518ef5c071c..debbde357c1a 100644 --- a/.circleci/src/jobs/create-sandboxes.yml +++ b/.circleci/src/jobs/create-sandboxes.yml @@ -51,7 +51,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - save_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} paths: - sandbox - store_test_results: diff --git a/.circleci/src/jobs/e2e-dev.yml b/.circleci/src/jobs/e2e-dev.yml index ab03d09cb266..312c141cf3d7 100644 --- a/.circleci/src/jobs/e2e-dev.yml +++ b/.circleci/src/jobs/e2e-dev.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/e2e-production.yml b/.circleci/src/jobs/e2e-production.yml index db9731ca1f28..eacf0bf4c4cd 100644 --- a/.circleci/src/jobs/e2e-production.yml +++ b/.circleci/src/jobs/e2e-production.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/test-runner-dev.yml b/.circleci/src/jobs/test-runner-dev.yml index 3842b9d38041..caea0d4fe62a 100644 --- a/.circleci/src/jobs/test-runner-dev.yml +++ b/.circleci/src/jobs/test-runner-dev.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Running Test Runner in Dev mode command: yarn task --task test-runner-dev --template $(yarn get-template --cadence << pipeline.parameters.workflow >> --task test-runner-dev) --no-link --start-from=never --junit diff --git a/.circleci/src/jobs/test-runner-production.yml b/.circleci/src/jobs/test-runner-production.yml index c2a29402f13f..ce2f5534fd0e 100644 --- a/.circleci/src/jobs/test-runner-production.yml +++ b/.circleci/src/jobs/test-runner-production.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | diff --git a/.circleci/src/jobs/vitest-integration.yml b/.circleci/src/jobs/vitest-integration.yml index b3e43ce1101c..fbfb7dfbdf79 100644 --- a/.circleci/src/jobs/vitest-integration.yml +++ b/.circleci/src/jobs/vitest-integration.yml @@ -26,7 +26,7 @@ steps: echo "sandbox/$TEMPLATE/package.json" > .sandbox-package-path.txt echo "Computed sandbox template: $TEMPLATE" - restore_cache: - key: sandbox-{{ .BuildNum }}-{{ checksum ".sandbox-package-path.txt" }} + key: sandbox-{{ .Revision }}-{{ checksum ".sandbox-package-path.txt" }} - run: name: Install sandbox dependencies command: | From c6975ee84c3e505e6962e9090ec60a50390214cc Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 10:42:01 +0000 Subject: [PATCH 6/7] Refactor getTemplate function to streamline template key retrieval and improve parallelism validation --- scripts/get-template.ts | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/scripts/get-template.ts b/scripts/get-template.ts index df2a503998e0..f0ba4cf5bd8f 100644 --- a/scripts/get-template.ts +++ b/scripts/get-template.ts @@ -49,32 +49,16 @@ export async function getTemplate( scriptName: string, { index, total }: { index: number; total: number } ) { - let potentialTemplateKeys: TemplateKey[] = []; - const isLocalSandbox = await pathExists(sandboxDir); - if (isLocalSandbox) { - const sandboxes = await getDirectories(sandboxDir); - potentialTemplateKeys = sandboxes - .map((dirName) => { - return Object.keys(allTemplates).find( - (templateKey) => templateKey.replace('/', '-') === dirName - ); - }) - .filter(Boolean) as TemplateKey[]; - } - - if (potentialTemplateKeys.length === 0) { - const cadenceTemplates = Object.entries(allTemplates).filter(([key]) => - templatesByCadence[cadence].includes(key as TemplateKey) - ); - potentialTemplateKeys = cadenceTemplates.map(([k]) => k) as TemplateKey[]; - } + const cadenceTemplates = Object.entries(allTemplates).filter(([key]) => + templatesByCadence[cadence].includes(key as TemplateKey) + ); - potentialTemplateKeys = potentialTemplateKeys.filter((t) => { + const potentialTemplateKeys = (cadenceTemplates.map(([k]) => k) as TemplateKey[]).filter((t) => { const currentTemplate = allTemplates[t] as Template; return isTaskSkipped(currentTemplate, scriptName); }); - if (!isLocalSandbox && potentialTemplateKeys.length !== total) { + if (potentialTemplateKeys.length !== total) { throw new Error(dedent`Circle parallelism set incorrectly. Parallelism is set to ${total}, but there are ${ From 80195fa59cb0c7be67690be0a8298957655d53b5 Mon Sep 17 00:00:00 2001 From: Valentin Palkovic Date: Mon, 24 Nov 2025 11:50:11 +0000 Subject: [PATCH 7/7] Remove unused imports and functions to streamline getTemplate script --- scripts/get-template.ts | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/scripts/get-template.ts b/scripts/get-template.ts index f0ba4cf5bd8f..f8dd1aa1cdcf 100644 --- a/scripts/get-template.ts +++ b/scripts/get-template.ts @@ -1,4 +1,4 @@ -import { access, readFile, readdir, writeFile } from 'node:fs/promises'; +import { readFile, writeFile } from 'node:fs/promises'; import { program } from 'commander'; import picocolors from 'picocolors'; @@ -12,30 +12,12 @@ import { allTemplates, templatesByCadence, } from '../code/lib/cli-storybook/src/sandbox-templates'; -import { SANDBOX_DIRECTORY } from './utils/constants'; import { esMain } from './utils/esmain'; -const sandboxDir = process.env.SANDBOX_ROOT || SANDBOX_DIRECTORY; - type Template = Pick; export type TemplateKey = keyof typeof allTemplates; export type Templates = Record; -async function getDirectories(source: string) { - return (await readdir(source, { withFileTypes: true })) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name); -} - -async function pathExists(path: string) { - try { - await access(path); - return true; - } catch { - return false; - } -} - function isTaskSkipped(template: Template, script: string): boolean { return ( template.inDevelopment !== true &&