test(integration): migrate flaky E2E tests to fake-openai-server

.github/workflows/e2e.yml

@@ -59,10 +59,10 @@ jobs:
           - '22.x'
     steps:
       - name: 'Checkout'
-        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10'  # v6.0.3
+        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3
 
       - name: 'Set up Node.js ${{ matrix.node-version }}'
-        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e'  # v6.4.0
+        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
         with:
           node-version: '${{ matrix.node-version }}'
           cache: 'npm'
@@ -139,9 +139,9 @@ jobs:
           # test:integration:sandbox:docker: that script would rebuild the image
           # the step above just built.
           if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
-            npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --shard='${{ matrix.shard }}'
+            npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
           else
-            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --shard='${{ matrix.shard }}'
+            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
           fi
 
   e2e-test-macos:
@@ -160,10 +160,10 @@ jobs:
           - '2/2'
     steps:
       - name: 'Checkout'
-        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10'  # v6.0.3
+        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3
 
       - name: 'Set up Node.js'
-        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e'  # v6.4.0
+        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -198,23 +198,28 @@ jobs:
           OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
           OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
           OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
-        run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --shard="${{ matrix.shard }}"'
+        run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --shard="${{ matrix.shard }}"'
 
-  cron-interactive-nightly:
-    name: 'cron-interactive E2E (nightly)'
+  isolated-nightly:
+    name: '${{ matrix.label }} (nightly)'
     runs-on: 'ubuntu-latest'
     if: |-
       ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
-    # This test is inherently timing-flaky (wall-clock cron fire + real model
-    # latency). Run it nightly only so flakes do not turn push CI red.
-    # continue-on-error prevents this job from marking the workflow as failed.
     continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: 'cron-interactive E2E'
+            test_file: 'interactive/cron-interactive.test.ts'
+          - label: 'channel-plugin E2E'
+            test_file: 'channel-plugin.test.ts'
     steps:
       - name: 'Checkout'
-        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10'  # v6.0.3
+        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3
 
       - name: 'Set up Node.js'
-        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e'  # v6.4.0
+        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
@@ -244,14 +249,14 @@ jobs:
         run: |-
           npm run bundle
 
-      - name: 'Run cron-interactive E2E tests'
+      - name: 'Run ${{ matrix.label }} tests'
         env:
           OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
           OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
           OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
           KEEP_OUTPUT: 'true'
           VERBOSE: 'true'
-        run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests interactive/cron-interactive.test.ts'
+        run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests "${{ matrix.test_file }}"'
 
   web-shell-browser-regression:
     name: 'web-shell Browser Regression'

integration-tests/cli/list_directory.test.ts

@@ -4,69 +4,116 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, afterEach, vi } from 'vitest';
 import {
+  fakeServerHostOptions,
+  IS_CONTAINER_SANDBOX,
+  CONTAINER_SANDBOX_NO_PROXY,
   TestRig,
-  printDebugInfo,
-  validateModelOutput,
 } from '../test-helper.js';
+import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 
 describe('list_directory', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
   it('should be able to list a directory', async () => {
     const rig = new TestRig();
     await rig.setup('should be able to list a directory');
     rig.createFile('file1.txt', 'file 1 content');
     rig.mkdir('subdir');
     rig.sync();
 
-    // Poll for filesystem changes to propagate in containers
     await rig.poll(
       () => {
-        // Check if the files exist in the test directory
         const file1Path = join(rig.testDir!, 'file1.txt');
         const subdirPath = join(rig.testDir!, 'subdir');
         return existsSync(file1Path) && existsSync(subdirPath);
       },
-      1000, // 1 second max wait
-      50, // check every 50ms
+      1000,
+      50,
     );
 
-    const prompt = `Call the list_directory tool on the current directory. You must use the tool — do not answer from the folder structure in your context.`;
+    const noProxy = IS_CONTAINER_SANDBOX
+      ? CONTAINER_SANDBOX_NO_PROXY
+      : '127.0.0.1,localhost';
 
-    const result = await rig.run(prompt);
+    let streamingRequestIndex = 0;
+    const fakeServer = await startFakeOpenAIServer(({ body }) => {
+      if (body['stream'] !== true) {
+        return { content: '{"selected_memories":[]}' };
+      }
+      const requestIndex = streamingRequestIndex++;
+      if (requestIndex === 0) {
+        return {
+          toolCalls: [
+            fakeToolCall('list_directory', { path: rig.testDir! }, 'list-dir'),
+          ],
+        };
+      }
+      return { content: 'The directory contains file1.txt and subdir.' };
+    }, fakeServerHostOptions());
 
-    const foundToolCall = await rig.waitForToolCall('list_directory');
+    vi.stubEnv('OPENAI_API_KEY', 'fake-key');
+    vi.stubEnv('OPENAI_BASE_URL', fakeServer.baseUrl);
+    vi.stubEnv('OPENAI_MODEL', 'fake-model');
+    vi.stubEnv('QWEN_MODEL', 'fake-model');
+    vi.stubEnv('QWEN_HOME', join(rig.testDir!, '.qwen-home'));
+    vi.stubEnv('QWEN_RUNTIME_DIR', join(rig.testDir!, '.qwen-home'));
+    vi.stubEnv('NO_PROXY', noProxy);
+    vi.stubEnv('no_proxy', noProxy);
 
-    // The model sometimes answers from the folder structure already present in
-    // the system prompt instead of calling the tool. Accept either a tool call
-    // OR correct text output so the test doesn't flake on model variability.
-    const hasCorrectOutput =
-      result.includes('file1.txt') && result.includes('subdir');
+    try {
+      const prompt = `Call the list_directory tool on the current directory.`;
+      // Explicit CLI flags outrank a developer's ~/.qwen/settings.json
+      // (settings.model.name beats the OPENAI_MODEL env var and can silently
+      // route the run to a real model endpoint instead of the fake server).
+      await rig.run(
+        prompt,
+        '--auth-type',
+        'openai',
+        '--model',
+        'fake-model',
+        '--openai-base-url',
+        fakeServer.baseUrl,
+        '--openai-api-key',
+        'fake-key',
+      );
 
-    // Add debugging information
-    if (!foundToolCall && !hasCorrectOutput) {
-      const allTools = printDebugInfo(rig, result, {
-        'Found tool call': foundToolCall,
-        'Contains file1.txt': result.includes('file1.txt'),
-        'Contains subdir': result.includes('subdir'),
-      });
+      const foundToolCall = await rig.waitForToolCall('list_directory');
 
-      console.error(
-        'List directory calls:',
-        allTools
-          .filter((t) => t.toolRequest.name === 'list_directory')
-          .map((t) => t.toolRequest.args),
+      expect(foundToolCall, 'Expected a list_directory tool call').toBe(true);
+
+      const toolResultRequest = fakeServer.requests.find(({ body }) => {
+        const messages = body['messages'];
+        return (
+          Array.isArray(messages) &&
+          messages.some(
+            (message) =>
+              typeof message === 'object' &&
+              message !== null &&
+              'role' in message &&
+              message.role === 'tool',
+          )
+        );
+      });
+      expect(
+        toolResultRequest,
+        'Expected a model request containing the list_directory result',
+      ).toBeDefined();
+      const messages = toolResultRequest?.body['messages'] as
+        | Array<{ role?: string; content?: unknown }>
+        | undefined;
+      const toolResultContent = JSON.stringify(
+        messages?.find((message) => message.role === 'tool')?.content ?? '',
       );
+      expect(toolResultContent).toContain('file1.txt');
+      expect(toolResultContent).toContain('subdir');
+    } finally {
+      await fakeServer.close();
     }
-
-    expect(
-      foundToolCall || hasCorrectOutput,
-      'Expected a list_directory tool call or correct directory listing in output',
-    ).toBeTruthy();
-
-    // Validate model output - will throw if no output, warn if missing expected content
-    validateModelOutput(result, ['file1.txt', 'subdir'], 'List directory test');
   });
 });