Generalize evals infra to support more types of evals, organization and queuing of named suites

.github/workflows/chained_e2e.yml

@@ -335,6 +335,8 @@ jobs:
         env:
           GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
           GEMINI_MODEL: 'gemini-3-pro-preview'
+          # Only run always passes behavioral tests.
+          EVAL_SUITE_TYPE: 'behavioral'
           # Disable Vitest internal retries to avoid double-retrying;
           # custom retry logic is handled in evals/test-helper.ts
           VITEST_RETRY: 0

.github/workflows/evals-nightly.yml

@@ -5,10 +5,18 @@ on:
     - cron: '0 1 * * *' # Runs at 1 AM every day
   workflow_dispatch:
     inputs:
-      run_all:
-        description: 'Run all evaluations (including usually passing)'
-        type: 'boolean'
-        default: true
+      suite_type:
+        description: 'Suite type to run'
+        type: 'choice'
+        options:
+          - 'behavioral'
+          - 'component-level'
+          - 'hero-scenario'
+        default: 'behavioral'
+      suite_name:
+        description: 'Specific suite name to run'
+        required: false
+        type: 'string'
       test_name_pattern:
         description: 'Test name pattern or file name'
         required: false
@@ -59,7 +67,9 @@ jobs:
         env:
           GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
           GEMINI_MODEL: '${{ matrix.model }}'
-          RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
+          RUN_EVALS: 'true'
+          EVAL_SUITE_TYPE: "${{ github.event.inputs.suite_type || 'behavioral' }}"
+          EVAL_SUITE_NAME: '${{ github.event.inputs.suite_name }}'
           TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
           # Disable Vitest internal retries to avoid double-retrying;
           # custom retry logic is handled in evals/test-helper.ts

evals/answer-vs-act.eval.ts

@@ -19,6 +19,8 @@ describe('Answer vs. ask eval', () => {
    * automatically modify the file, but instead asks for permission.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should not edit files when asked to inspect for bugs',
     prompt: 'Inspect app.ts for bugs',
     files: FILES,
@@ -42,6 +44,8 @@ describe('Answer vs. ask eval', () => {
    * does modify the file.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should edit files when asked to fix bug',
     prompt: 'Fix the bug in app.ts - it should add numbers not subtract',
     files: FILES,
@@ -66,6 +70,8 @@ describe('Answer vs. ask eval', () => {
    * automatically modify the file, but instead asks for permission.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should not edit when asking "any bugs"',
     prompt: 'Any bugs in app.ts?',
     files: FILES,
@@ -89,6 +95,8 @@ describe('Answer vs. ask eval', () => {
    * automatically modify the file.
    */
   evalTest('ALWAYS_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should not edit files when asked a general question',
     prompt: 'How does app.ts work?',
     files: FILES,
@@ -112,6 +120,8 @@ describe('Answer vs. ask eval', () => {
    * automatically modify the file.
    */
   evalTest('ALWAYS_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should not edit files when asked about style',
     prompt: 'Is app.ts following good style?',
     files: FILES,
@@ -135,6 +145,8 @@ describe('Answer vs. ask eval', () => {
    * the agent does NOT automatically modify the file.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should not edit files when user notes an issue',
     prompt: 'The add function subtracts numbers.',
     files: FILES,

evals/app-test-helper.ts

@@ -10,10 +10,13 @@ import {
   runEval,
   prepareLogDir,
   symlinkNodeModules,
+  withEvalRetries,
+  prepareWorkspace,
+  type BaseEvalCase,
+  EVAL_MODEL,
 } from './test-helper.js';
 import fs from 'node:fs';
 import path from 'node:path';
-import { DEFAULT_GEMINI_MODEL } from '@google/gemini-cli-core';
 
 /**
  * Config overrides for evals, with tool-restriction fields explicitly
@@ -29,15 +32,13 @@ interface EvalConfigOverrides {
   allowedTools?: never;
   /** Restricting tools via mainAgentTools in evals is forbidden. */
   mainAgentTools?: never;
+
   [key: string]: unknown;
 }
 
-export interface AppEvalCase {
-  name: string;
+export interface AppEvalCase extends BaseEvalCase {
   configOverrides?: EvalConfigOverrides;
   prompt: string;
-  timeout?: number;
-  files?: Record<string, string>;
   setup?: (rig: AppRig) => Promise<void>;
   assert: (rig: AppRig, output: string) => Promise<void>;
 }
@@ -48,56 +49,55 @@ export interface AppEvalCase {
  */
 export function appEvalTest(policy: EvalPolicy, evalCase: AppEvalCase) {
   const fn = async () => {
-    const rig = new AppRig({
-      configOverrides: {
-        model: DEFAULT_GEMINI_MODEL,
-        ...evalCase.configOverrides,
-      },
-    });
+    await withEvalRetries(evalCase.name, async () => {
+      const rig = new AppRig({
+        configOverrides: {
+          model: EVAL_MODEL,
+          ...evalCase.configOverrides,
+        },
+      });
 
-    const { logDir, sanitizedName } = await prepareLogDir(evalCase.name);
-    const logFile = path.join(logDir, `${sanitizedName}.log`);
+      const { logDir, sanitizedName } = await prepareLogDir(evalCase.name);
+      const logFile = path.join(logDir, `${sanitizedName}.log`);
 
-    try {
-      await rig.initialize();
+      try {
+        await rig.initialize();
 
-      const testDir = rig.getTestDir();
-      symlinkNodeModules(testDir);
+        const testDir = rig.getTestDir();
+        symlinkNodeModules(testDir);
 
-      // Setup initial files
-      if (evalCase.files) {
-        for (const [filePath, content] of Object.entries(evalCase.files)) {
-          const fullPath = path.join(testDir, filePath);
-          fs.mkdirSync(path.dirname(fullPath), { recursive: true });
-          fs.writeFileSync(fullPath, content);
+        // Setup initial files
+        if (evalCase.files) {
+          // Note: AppRig does not use a separate homeDir, so we use testDir twice
+          await prepareWorkspace(testDir, testDir, evalCase.files);
         }
-      }
 
-      // Run custom setup if provided (e.g. for breakpoints)
-      if (evalCase.setup) {
-        await evalCase.setup(rig);
-      }
+        // Run custom setup if provided (e.g. for breakpoints)
+        if (evalCase.setup) {
+          await evalCase.setup(rig);
+        }
 
-      // Render the app!
-      await rig.render();
+        // Render the app!
+        await rig.render();
 
-      // Wait for initial ready state
-      await rig.waitForIdle();
+        // Wait for initial ready state
+        await rig.waitForIdle();
 
-      // Send the initial prompt
-      await rig.sendMessage(evalCase.prompt);
+        // Send the initial prompt
+        await rig.sendMessage(evalCase.prompt);
 
-      // Run assertion. Interaction-heavy tests can do their own waiting/steering here.
-      const output = rig.getStaticOutput();
-      await evalCase.assert(rig, output);
-    } finally {
-      const output = rig.getStaticOutput();
-      if (output) {
-        await fs.promises.writeFile(logFile, output);
+        // Run assertion. Interaction-heavy tests can do their own waiting/steering here.
+        const output = rig.getStaticOutput();
+        await evalCase.assert(rig, output);
+      } finally {
+        const output = rig.getStaticOutput();
+        if (output) {
+          await fs.promises.writeFile(logFile, output);
+        }
+        await rig.unmount();
       }
-      await rig.unmount();
-    }
+    });
   };
 
-  runEval(policy, evalCase.name, fn, (evalCase.timeout ?? 60000) + 10000);
+  runEval(policy, evalCase, fn, (evalCase.timeout ?? 60000) + 10000);
 }

evals/ask_user.eval.ts

@@ -5,17 +5,21 @@
  */
 
 import { describe, expect } from 'vitest';
-import { appEvalTest, AppEvalCase } from './app-test-helper.js';
-import { EvalPolicy } from './test-helper.js';
+import { ApprovalMode, isRecord } from '@google/gemini-cli-core';
+import { appEvalTest, type AppEvalCase } from './app-test-helper.js';
+import { type EvalPolicy } from './test-helper.js';
 
 function askUserEvalTest(policy: EvalPolicy, evalCase: AppEvalCase) {
+  const existingGeneral = evalCase.configOverrides?.['general'];
+  const generalBase = isRecord(existingGeneral) ? existingGeneral : {};
+
   return appEvalTest(policy, {
     ...evalCase,
     configOverrides: {
       ...evalCase.configOverrides,
+      approvalMode: ApprovalMode.DEFAULT,
       general: {
-        ...evalCase.configOverrides?.general,
-        approvalMode: 'default',
+        ...generalBase,
         enableAutoUpdate: false,
         enableAutoUpdateNotification: false,
       },
@@ -28,6 +32,8 @@ function askUserEvalTest(policy: EvalPolicy, evalCase: AppEvalCase) {
 
 describe('ask_user', () => {
   askUserEvalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'Agent uses AskUser tool to present multiple choice options',
     prompt: `Use the ask_user tool to ask me what my favorite color is. Provide 3 options: red, green, or blue.`,
     setup: async (rig) => {
@@ -43,6 +49,8 @@ describe('ask_user', () => {
   });
 
   askUserEvalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'Agent uses AskUser tool to clarify ambiguous requirements',
     files: {
       'package.json': JSON.stringify({ name: 'my-app', version: '1.0.0' }),
@@ -61,6 +69,8 @@ describe('ask_user', () => {
   });
 
   askUserEvalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'Agent uses AskUser tool before performing significant ambiguous rework',
     files: {
       'packages/core/src/index.ts': '// index\nexport const version = "1.0.0";',
@@ -82,8 +92,8 @@ describe('ask_user', () => {
       ]);
       expect(confirmation, 'Expected a tool call confirmation').toBeDefined();
 
-      if (confirmation?.name === 'enter_plan_mode') {
-        rig.acceptConfirmation('enter_plan_mode');
+      if (confirmation?.toolName === 'enter_plan_mode') {
+        await rig.resolveTool('enter_plan_mode');
         confirmation = await rig.waitForPendingConfirmation('ask_user');
       }
 
@@ -101,6 +111,8 @@ describe('ask_user', () => {
   // updates to clarify that shell command confirmation is handled by the UI.
   // See fix: https://github.com/google-gemini/gemini-cli/pull/20504
   askUserEvalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'Agent does NOT use AskUser to confirm shell commands',
     files: {
       'package.json': JSON.stringify({

evals/automated-tool-use.eval.ts

@@ -14,6 +14,8 @@ describe('Automated tool use', () => {
    * a repro by guiding the agent into using the existing deficient script.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should use automated tools (eslint --fix) to fix code style issues',
     files: {
       'package.json': JSON.stringify(
@@ -102,6 +104,8 @@ describe('Automated tool use', () => {
    * instead of trying to edit the files itself.
    */
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should use automated tools (prettier --write) to fix formatting issues',
     files: {
       'package.json': JSON.stringify(

evals/cli_help_delegation.eval.ts

@@ -3,6 +3,8 @@ import { evalTest } from './test-helper.js';
 
 describe('CliHelpAgent Delegation', () => {
   evalTest('USUALLY_PASSES', {
+    suiteName: 'default',
+    suiteType: 'behavioral',
     name: 'should delegate to cli_help agent for subagent creation questions',
     params: {
       settings: {