From 193db287a6d2cbe462eb8b343bf7327f6d28200c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 04:14:27 +0000 Subject: [PATCH] ci: stop running evals automatically on every commit Evals were running on every PR commit (eval-regression.yaml on pull_request synchronize) and every push to master (eval-master.yaml), which was the main CI eval cost driver even when no one looked at the results. Switch both to on-demand: - eval-regression.yaml: trigger only on the `labeled` pull_request event instead of opened/synchronize/reopened/labeled, and skip by default in the automatic path unless the PR carries an evals-* label (evals-tag-*, evals-id-*, evals-model-*). /eval comments and workflow_dispatch are unaffected, so on-demand runs still work. - eval-master.yaml: drop the push-to-master trigger; refresh the baseline on demand via workflow_dispatch. The weekly benchmark cron (eval-benchmarks.yaml) is unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BzLq3Ubqo1TTonzkUASL3X Signed-off-by: Claude --- .github/workflows/eval-master.yaml | 13 +++---------- .github/workflows/eval-regression.yaml | 26 ++++++++++++++++++++------ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/eval-master.yaml b/.github/workflows/eval-master.yaml index 70fd394223..5cc4ff74ad 100644 --- a/.github/workflows/eval-master.yaml +++ b/.github/workflows/eval-master.yaml @@ -1,16 +1,9 @@ name: Run eval regression on master on: - # Run on every push to master so PRs can compare against the most recent - # known-good state of master, not just the weekly benchmark. - push: - branches: [master] - paths-ignore: - - 'docs/**' - - '**/*.md' - - 'mkdocs.yml' - - # Allow manual triggering for one-off baselines / re-running after a fix. + # The master baseline no longer runs automatically on every push to master — + # that per-merge eval cost has been removed. Refresh the baseline on demand via + # workflow_dispatch (e.g. one-off baselines or re-running after a fix). workflow_dispatch: inputs: models: diff --git a/.github/workflows/eval-regression.yaml b/.github/workflows/eval-regression.yaml index a0909fb315..08733e92e2 100644 --- a/.github/workflows/eval-regression.yaml +++ b/.github/workflows/eval-regression.yaml @@ -1,9 +1,12 @@ name: Run eval regression tests on: + # Evals no longer run automatically on every commit (this was the main CI eval + # cost driver). They run on-demand only: add an evals-* label to a PR (handled + # by the `labeled` event below), comment `/eval`, or trigger workflow_dispatch. pull_request: branches: ["*"] - types: [opened, synchronize, reopened, labeled] + types: [labeled] paths-ignore: - 'docs/**' - '**/*.md' @@ -452,6 +455,12 @@ jobs: prNumber = event === 'pull_request' ? context.payload.pull_request.number : null; triggeredBy = ''; + // Evals do not run automatically on every commit anymore. For automatic + // (pull_request) triggers we skip by default and only run when the PR has + // at least one evals-* label opting it in (evals-tag-*, evals-id-*, or + // evals-model-*). /eval comments and workflow_dispatch are unaffected. + skipEval = true; + // Parse PR labels for evals-tag-*, evals-id-*, and evals-model-* to augment the regression run if (event === 'pull_request' && context.payload.pull_request.labels) { const labelSafe = /^[a-zA-Z0-9_\-]+$/; @@ -480,22 +489,27 @@ jobs: // Only IDs specified: broaden markers to all LLM tests, filter by ID markers = ''; filter = idLabels.join(' or '); + skipEval = false; } else if (tagLabels.length > 0 && idLabels.length === 0) { // Only tags specified: add to regression markers (unless skip-default) markers = skipDefault ? tagLabels.join(' or ') : 'regression or ' + tagLabels.join(' or '); + skipEval = false; } else if (tagLabels.length > 0 && idLabels.length > 0) { // Both specified: use tag markers + ID filter markers = skipDefault ? tagLabels.join(' or ') : 'regression or ' + tagLabels.join(' or '); filter = idLabels.join(' or '); - } else if (skipDefault) { - // Only skip-default, no other evals-* labels: skip eval run entirely - skipEval = true; + skipEval = false; } - // else: no evals-* labels, keep defaults (markers = 'regression') + // else: no evals-tag-*/evals-id-* labels — stays skipped (skipEval=true) + // unless an evals-model-* label opts in below. evals-skip-default alone + // is now a no-op since skipping is the default. - // Override model if evals-model-* label is present + // Override model if evals-model-* label is present. A model label on + // its own means "run the default regression suite on this model", so it + // opts the PR in (markers falls through to the 'regression' default below). if (modelLabels.length > 0) { model = modelLabels.join(','); + skipEval = false; } }