diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 30a181ad0cf..744ebbbe515 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -59,10 +59,10 @@ jobs: - '22.x' steps: - name: 'Checkout' - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 - name: 'Set up Node.js ${{ matrix.node-version }}' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version: '${{ matrix.node-version }}' cache: 'npm' @@ -139,9 +139,9 @@ jobs: # test:integration:sandbox:docker: that script would rebuild the image # the step above just built. if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then - npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --shard='${{ matrix.shard }}' + npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' else - npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --shard='${{ matrix.shard }}' + npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}' fi e2e-test-macos: @@ -160,10 +160,10 @@ jobs: - '2/2' steps: - name: 'Checkout' - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 - name: 'Set up Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -198,23 +198,28 @@ jobs: OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}' OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}' - run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --shard="${{ matrix.shard }}"' + run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --shard="${{ matrix.shard }}"' - cron-interactive-nightly: - name: 'cron-interactive E2E (nightly)' + isolated-nightly: + name: '${{ matrix.label }} (nightly)' runs-on: 'ubuntu-latest' if: |- ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - # This test is inherently timing-flaky (wall-clock cron fire + real model - # latency). Run it nightly only so flakes do not turn push CI red. - # continue-on-error prevents this job from marking the workflow as failed. continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - label: 'cron-interactive E2E' + test_file: 'interactive/cron-interactive.test.ts' + - label: 'channel-plugin E2E' + test_file: 'channel-plugin.test.ts' steps: - name: 'Checkout' - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 - name: 'Set up Node.js' - uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 + uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' cache: 'npm' @@ -244,14 +249,14 @@ jobs: run: |- npm run bundle - - name: 'Run cron-interactive E2E tests' + - name: 'Run ${{ matrix.label }} tests' env: OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}' OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}' KEEP_OUTPUT: 'true' VERBOSE: 'true' - run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests interactive/cron-interactive.test.ts' + run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests "${{ matrix.test_file }}"' web-shell-browser-regression: name: 'web-shell Browser Regression' diff --git a/integration-tests/cli/list_directory.test.ts b/integration-tests/cli/list_directory.test.ts index ad488f4aef3..3c724ebf06c 100644 --- a/integration-tests/cli/list_directory.test.ts +++ b/integration-tests/cli/list_directory.test.ts @@ -4,16 +4,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, afterEach, vi } from 'vitest'; import { + fakeServerHostOptions, + IS_CONTAINER_SANDBOX, + CONTAINER_SANDBOX_NO_PROXY, TestRig, - printDebugInfo, - validateModelOutput, } from '../test-helper.js'; +import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js'; import { existsSync } from 'node:fs'; import { join } from 'node:path'; describe('list_directory', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it('should be able to list a directory', async () => { const rig = new TestRig(); await rig.setup('should be able to list a directory'); @@ -21,52 +27,93 @@ describe('list_directory', () => { rig.mkdir('subdir'); rig.sync(); - // Poll for filesystem changes to propagate in containers await rig.poll( () => { - // Check if the files exist in the test directory const file1Path = join(rig.testDir!, 'file1.txt'); const subdirPath = join(rig.testDir!, 'subdir'); return existsSync(file1Path) && existsSync(subdirPath); }, - 1000, // 1 second max wait - 50, // check every 50ms + 1000, + 50, ); - const prompt = `Call the list_directory tool on the current directory. You must use the tool — do not answer from the folder structure in your context.`; + const noProxy = IS_CONTAINER_SANDBOX + ? CONTAINER_SANDBOX_NO_PROXY + : '127.0.0.1,localhost'; - const result = await rig.run(prompt); + let streamingRequestIndex = 0; + const fakeServer = await startFakeOpenAIServer(({ body }) => { + if (body['stream'] !== true) { + return { content: '{"selected_memories":[]}' }; + } + const requestIndex = streamingRequestIndex++; + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall('list_directory', { path: rig.testDir! }, 'list-dir'), + ], + }; + } + return { content: 'The directory contains file1.txt and subdir.' }; + }, fakeServerHostOptions()); - const foundToolCall = await rig.waitForToolCall('list_directory'); + vi.stubEnv('OPENAI_API_KEY', 'fake-key'); + vi.stubEnv('OPENAI_BASE_URL', fakeServer.baseUrl); + vi.stubEnv('OPENAI_MODEL', 'fake-model'); + vi.stubEnv('QWEN_MODEL', 'fake-model'); + vi.stubEnv('QWEN_HOME', join(rig.testDir!, '.qwen-home')); + vi.stubEnv('QWEN_RUNTIME_DIR', join(rig.testDir!, '.qwen-home')); + vi.stubEnv('NO_PROXY', noProxy); + vi.stubEnv('no_proxy', noProxy); - // The model sometimes answers from the folder structure already present in - // the system prompt instead of calling the tool. Accept either a tool call - // OR correct text output so the test doesn't flake on model variability. - const hasCorrectOutput = - result.includes('file1.txt') && result.includes('subdir'); + try { + const prompt = `Call the list_directory tool on the current directory.`; + // Explicit CLI flags outrank a developer's ~/.qwen/settings.json + // (settings.model.name beats the OPENAI_MODEL env var and can silently + // route the run to a real model endpoint instead of the fake server). + await rig.run( + prompt, + '--auth-type', + 'openai', + '--model', + 'fake-model', + '--openai-base-url', + fakeServer.baseUrl, + '--openai-api-key', + 'fake-key', + ); - // Add debugging information - if (!foundToolCall && !hasCorrectOutput) { - const allTools = printDebugInfo(rig, result, { - 'Found tool call': foundToolCall, - 'Contains file1.txt': result.includes('file1.txt'), - 'Contains subdir': result.includes('subdir'), - }); + const foundToolCall = await rig.waitForToolCall('list_directory'); - console.error( - 'List directory calls:', - allTools - .filter((t) => t.toolRequest.name === 'list_directory') - .map((t) => t.toolRequest.args), + expect(foundToolCall, 'Expected a list_directory tool call').toBe(true); + + const toolResultRequest = fakeServer.requests.find(({ body }) => { + const messages = body['messages']; + return ( + Array.isArray(messages) && + messages.some( + (message) => + typeof message === 'object' && + message !== null && + 'role' in message && + message.role === 'tool', + ) + ); + }); + expect( + toolResultRequest, + 'Expected a model request containing the list_directory result', + ).toBeDefined(); + const messages = toolResultRequest?.body['messages'] as + | Array<{ role?: string; content?: unknown }> + | undefined; + const toolResultContent = JSON.stringify( + messages?.find((message) => message.role === 'tool')?.content ?? '', ); + expect(toolResultContent).toContain('file1.txt'); + expect(toolResultContent).toContain('subdir'); + } finally { + await fakeServer.close(); } - - expect( - foundToolCall || hasCorrectOutput, - 'Expected a list_directory tool call or correct directory listing in output', - ).toBeTruthy(); - - // Validate model output - will throw if no output, warn if missing expected content - validateModelOutput(result, ['file1.txt', 'subdir'], 'List directory test'); }); }); diff --git a/integration-tests/sdk-typescript/abort-and-lifecycle.test.ts b/integration-tests/sdk-typescript/abort-and-lifecycle.test.ts index 4f5d0092f07..e6276650058 100644 --- a/integration-tests/sdk-typescript/abort-and-lifecycle.test.ts +++ b/integration-tests/sdk-typescript/abort-and-lifecycle.test.ts @@ -21,16 +21,46 @@ import { createSharedTestOptions, createResultWaiter, } from './test-helper.js'; +import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js'; +import { + IS_CONTAINER_SANDBOX, + CONTAINER_SANDBOX_NO_PROXY, + fakeServerHostOptions, +} from '../test-helper.js'; const SHARED_TEST_OPTIONS = createSharedTestOptions(); +const LOCAL_OPENAI_NO_PROXY = IS_CONTAINER_SANDBOX + ? CONTAINER_SANDBOX_NO_PROXY + : '127.0.0.1,localhost'; +const FAKE_SERVER_OPTIONS = fakeServerHostOptions(); + +function fakeModelOptions(baseUrl: string) { + return { + model: 'fake-model', + authType: 'openai' as const, + env: { + NO_PROXY: LOCAL_OPENAI_NO_PROXY, + no_proxy: LOCAL_OPENAI_NO_PROXY, + OPENAI_API_KEY: 'fake-key', + OPENAI_BASE_URL: baseUrl, + OPENAI_MODEL: 'fake-model', + QWEN_MODEL: 'fake-model', + }, + }; +} + +const LONG_CONTENT_CHUNKS = Array.from({ length: 30 }, (_, i) => `chunk${i} `); + describe('AbortController and Process Lifecycle (E2E)', () => { let helper: SDKTestHelper; let testDir: string; beforeEach(async () => { helper = new SDKTestHelper(); - testDir = await helper.setup('abort-and-lifecycle'); + testDir = await helper.setup('abort-and-lifecycle', { + settings: { fastModel: 'openai:fake-model' }, + }); }); afterEach(async () => { @@ -41,11 +71,17 @@ describe('AbortController and Process Lifecycle (E2E)', () => { const controller = new AbortController(); const TARGET_CHARS = 50; let accumulatedText = ''; + let messagesAfterAbort = 0; + + const fakeServer = await startFakeOpenAIServer(() => { + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); const q = query({ prompt: 'Write a very long story about TypeScript programming', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, includePartialMessages: true, @@ -55,6 +91,9 @@ describe('AbortController and Process Lifecycle (E2E)', () => { try { for await (const message of q) { + if (controller.signal.aborted) { + messagesAfterAbort++; + } if (isSDKPartialAssistantMessage(message)) { // Handle partial messages from streaming if ( @@ -89,18 +128,25 @@ describe('AbortController and Process Lifecycle (E2E)', () => { expect(isAbortError(error)).toBe(true); // Should have accumulated at least TARGET_CHARS before abort expect(accumulatedText.length).toBeGreaterThanOrEqual(TARGET_CHARS); + expect(messagesAfterAbort).toBeLessThan(10); } finally { await q.close(); + await fakeServer.close(); } }); it('should handle abort during query execution', async () => { const controller = new AbortController(); + const fakeServer = await startFakeOpenAIServer(() => { + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -119,6 +165,9 @@ describe('AbortController and Process Lifecycle (E2E)', () => { } } } + + // Should not reach here - query should be aborted + expect(false).toBe(true); } catch (error) { expect(isAbortError(error)).toBe(true); expect(error instanceof AbortError).toBe(true); @@ -126,16 +175,25 @@ describe('AbortController and Process Lifecycle (E2E)', () => { expect(receivedFirstMessage).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); it('should handle abort immediately after query starts', async () => { const controller = new AbortController(); + // Hold the model response open so the 200ms abort below lands inside + // the model request instead of racing CLI startup. + const fakeServer = await startFakeOpenAIServer(async () => { + await new Promise((resolve) => setTimeout(resolve, 5000)); + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Write a very long essay', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -151,21 +209,30 @@ describe('AbortController and Process Lifecycle (E2E)', () => { for await (const _message of q) { // May or may not receive messages before abort } + + // Should not reach here - query should be aborted + expect(false).toBe(true); } catch (error) { expect(isAbortError(error)).toBe(true); expect(error instanceof AbortError).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); }); describe('Process Lifecycle Monitoring', () => { it('should handle normal process completion', async () => { + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello! How can I help you?' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Say hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: false, }, @@ -187,50 +254,56 @@ describe('AbortController and Process Lifecycle (E2E)', () => { expect(false).toBe(true); } finally { await q.close(); + await fakeServer.close(); expect(completedSuccessfully).toBe(true); expect(receivedAssistantMessage).toBe(true); } }); it('should handle process cleanup after error', async () => { + const controller = new AbortController(); + const fakeServer = await startFakeOpenAIServer(() => { + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Hello world', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, + abortController: controller, debug: false, }, }); + let receivedError = false; try { - for await (const message of q) { - if (isSDKAssistantMessage(message)) { - const textBlocks = message.message.content.filter( - (block): block is TextBlock => block.type === 'text', - ); - const text = textBlocks - .map((b) => b.text) - .join('') - .slice(0, 50); - expect(text.length).toBeGreaterThan(0); - } + for await (const _message of q) { + controller.abort(); } } catch (error) { - // Expected to potentially have errors + receivedError = true; + expect(isAbortError(error)).toBe(true); } finally { - // Should cleanup successfully even after error await q.close(); - expect(true).toBe(true); // Cleanup completed + await fakeServer.close(); + expect(receivedError).toBe(true); } }); }); describe('Input Stream Control', () => { it('should support endInput() method', async () => { + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello! How can I help you?' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Say hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: false, }, @@ -254,12 +327,17 @@ describe('AbortController and Process Lifecycle (E2E)', () => { expect(endInputCalled).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); }); describe('Closed stdin behavior (asyncGenerator prompt)', () => { it('should reject control requests after stdin closes', async () => { + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'OK' }; + }, FAKE_SERVER_OPTIONS); + const resultWaiter = createResultWaiter(1); let promptDoneResolve: () => void = () => {}; const promptDonePromise = new Promise((resolve) => { @@ -285,6 +363,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { prompt: createPrompt(), options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: false, }, @@ -310,6 +389,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { ); } finally { await q.close(); + await fakeServer.close(); } }); @@ -317,6 +397,52 @@ describe('AbortController and Process Lifecycle (E2E)', () => { const testFilePath = await helper.getPath('test.txt'); await helper.createFile('test.txt', 'original content'); + const fakeServer = await startFakeOpenAIServer(({ body }) => { + const messages = (body['messages'] ?? []) as Array< + Record + >; + const transcript = JSON.stringify(messages); + // The second user turn is identified by a quote-free marker: the + // prompt contains `"updated"`, and JSON.stringify escapes those + // quotes, so a needle containing a raw `"` can never match. + if (!transcript.includes('Stop if any exception occurs')) { + return { content: 'OK' }; + } + const alreadyCalled = (name: string) => + messages.some( + (m) => + Array.isArray(m['tool_calls']) && + (m['tool_calls'] as Array<{ function?: { name?: string } }>).some( + (t) => t.function?.name === name, + ), + ); + // write_file refuses to overwrite a file that has not been read in + // this session, so the script has to read before it writes. + if (!alreadyCalled('read_file')) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: testFilePath }, + 'read-target', + ), + ], + }; + } + if (!alreadyCalled('write_file')) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { file_path: testFilePath, content: 'updated' }, + 'write-target', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + // Bounded promise with explicit timer arming and clearing on settle. // `startTimer()` lets each phase begin counting only when its phase // actually starts, so slow predecessors don't burn its budget and @@ -406,6 +532,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { prompt: createPrompt(), options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', coreTools: ['read_file', 'write_file'], @@ -471,6 +598,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { } finally { for (const t of pendingTimers) t.clear(); await q.close(); + await fakeServer.close(); } }); }); @@ -505,10 +633,18 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should throw AbortError with correct properties', async () => { const controller = new AbortController(); + // Hold the model response open so the 1000ms abort below lands inside + // the model request instead of racing CLI startup. + const fakeServer = await startFakeOpenAIServer(async () => { + await new Promise((resolve) => setTimeout(resolve, 5000)); + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Explain the concept of async programming', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -522,6 +658,9 @@ describe('AbortController and Process Lifecycle (E2E)', () => { for await (const _message of q) { // May receive some messages before abort } + + // Should not reach here - query should be aborted + expect(false).toBe(true); } catch (error) { // Verify error type and helper functions expect(isAbortError(error)).toBe(true); @@ -530,6 +669,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { expect((error as Error).message).toBeDefined(); } finally { await q.close(); + await fakeServer.close(); } }); }); @@ -538,10 +678,15 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should capture stderr messages when debug is enabled', async () => { const stderrMessages: string[] = []; + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello!' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Say hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: true, stderr: (msg: string) => { @@ -556,6 +701,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); expect(stderrMessages.length).toBeGreaterThan(0); } }); @@ -563,10 +709,15 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should not capture stderr when debug is disabled', async () => { const stderrMessages: string[] = []; + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello!' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: false, stderr: (msg: string) => { @@ -581,6 +732,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); // Should have minimal or no stderr output when debug is false expect(stderrMessages.length).toBeLessThan(10); } @@ -591,10 +743,15 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should cleanup properly after abort', async () => { const controller = new AbortController(); + const fakeServer = await startFakeOpenAIServer(() => { + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Write a very long essay about programming', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -616,6 +773,7 @@ describe('AbortController and Process Lifecycle (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); expect(true).toBe(true); // Cleanup completed after abort } }); @@ -623,10 +781,18 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should handle multiple abort calls gracefully', async () => { const controller = new AbortController(); + // Hold the model response open so the timer aborts below land inside + // the model request instead of racing CLI startup. + const fakeServer = await startFakeOpenAIServer(async () => { + await new Promise((resolve) => setTimeout(resolve, 5000)); + return { contentChunks: LONG_CONTENT_CHUNKS }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Count to 100', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -642,20 +808,29 @@ describe('AbortController and Process Lifecycle (E2E)', () => { for await (const _message of q) { // Should be interrupted } + + // Should not reach here - query should be aborted + expect(false).toBe(true); } catch (error) { expect(isAbortError(error)).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); }); describe('Resource Management Edge Cases', () => { it('should handle close() called multiple times', async () => { + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello!' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, debug: false, }, @@ -665,10 +840,14 @@ describe('AbortController and Process Lifecycle (E2E)', () => { const iterator = q[Symbol.asyncIterator](); await iterator.next(); - // Close multiple times - await q.close(); - await q.close(); - await q.close(); + try { + // Close multiple times + await q.close(); + await q.close(); + await q.close(); + } finally { + await fakeServer.close(); + } // Should not throw expect(true).toBe(true); @@ -677,10 +856,15 @@ describe('AbortController and Process Lifecycle (E2E)', () => { it('should handle abort after close', async () => { const controller = new AbortController(); + const fakeServer = await startFakeOpenAIServer(() => { + return { content: 'Hello!' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Hello', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, abortController: controller, debug: false, @@ -690,13 +874,17 @@ describe('AbortController and Process Lifecycle (E2E)', () => { // Start and close immediately const iterator = q[Symbol.asyncIterator](); await iterator.next(); - await q.close(); + try { + await q.close(); - // Abort after close - controller.abort(); + // Abort after close + controller.abort(); - // Should not throw - expect(true).toBe(true); + // Should not throw + expect(true).toBe(true); + } finally { + await fakeServer.close(); + } }); }); }); diff --git a/integration-tests/sdk-typescript/tool-control.test.ts b/integration-tests/sdk-typescript/tool-control.test.ts index 8f8cbbd0602..983ac9d80fa 100644 --- a/integration-tests/sdk-typescript/tool-control.test.ts +++ b/integration-tests/sdk-typescript/tool-control.test.ts @@ -19,7 +19,11 @@ import { type SDKMessage, type SDKUserMessage, } from '@qwen-code/sdk'; -import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js'; +import { + fakeToolCall, + startFakeOpenAIServer, + type FakeOpenAIServer, +} from '../fake-openai-server.js'; import { SDKTestHelper, extractText, @@ -59,13 +63,28 @@ function fakeModelOptions(baseUrl: string) { }; } +function advertisedToolNames(fakeServer: FakeOpenAIServer): string[] { + const tools = fakeServer.requests.find(({ body }) => body['stream'] === true) + ?.body['tools']; + if (!Array.isArray(tools)) return []; + return tools.flatMap((tool): string[] => { + if (typeof tool !== 'object' || tool === null) return []; + const fn = (tool as { function?: unknown }).function; + if (typeof fn !== 'object' || fn === null) return []; + const name = (fn as { name?: unknown }).name; + return typeof name === 'string' ? [name] : []; + }); +} + describe('Tool Control Parameters (E2E)', () => { let helper: SDKTestHelper; let testDir: string; beforeEach(async () => { helper = new SDKTestHelper(); - testDir = await helper.setup('tool-control'); + testDir = await helper.setup('tool-control', { + settings: { fastModel: 'openai:fake-model' }, + }); }); afterEach(async () => { @@ -79,11 +98,36 @@ describe('Tool Control Parameters (E2E)', () => { // Create a test file await helper.createFile('test.txt', INITIAL_CONTENT); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + fakeToolCall('list_directory', { path: testDir }, 'list-dir'), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read the file test.txt and then write "modified" to test.txt. Finally, list the directory.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Only allow read_file and write_file, exclude list_directory @@ -106,8 +150,21 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); expect(toolNames).toContain('write_file'); - // Should NOT have list_directory since it's not in coreTools - expect(toolNames).not.toContain('list_directory'); + const advertisedTools = advertisedToolNames(fakeServer); + expect(advertisedTools).toEqual( + expect.arrayContaining(['read_file', 'write_file']), + ); + + const listDirectoryResults = findToolResults( + messages, + 'list_directory', + ); + expect(listDirectoryResults).toHaveLength(1); + expect(listDirectoryResults[0]).toMatchObject({ + isError: true, + content: expect.stringContaining('was declined'), + }); + expect(advertisedTools).not.toContain('list_directory'); // Verify the write_file call itself requested different content // than the original. Asserting on the tool-call arguments (rather @@ -126,6 +183,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(writtenContent).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -134,10 +192,15 @@ describe('Tool Control Parameters (E2E)', () => { it( 'should work with minimal tool set', async () => { + const fakeServer = await startFakeOpenAIServer(() => { + return { content: '4' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'What is 2 + 2? Just answer with the number.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, // Only allow thinking, no file operations coreTools: [], @@ -167,6 +230,7 @@ describe('Tool Control Parameters (E2E)', () => { assertSuccessfulCompletion(messages); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -179,11 +243,35 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test content'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: '', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and then write empty content to it to clear it.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', coreTools: ['read_file', 'write_file'], @@ -223,6 +311,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toBe('test content'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -233,10 +322,32 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test content'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall('list_directory', { path: testDir }, 'list-dir'), + fakeToolCall( + 'run_shell_command', + { command: 'echo hello' }, + 'shell-echo', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt, list the directory, and run "echo hello".', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Block multiple tools @@ -278,6 +389,7 @@ describe('Tool Control Parameters (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -286,10 +398,31 @@ describe('Tool Control Parameters (E2E)', () => { it( 'should block all shell commands when run_shell_command is excluded', async () => { + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'run_shell_command', + { command: 'echo hello' }, + 'shell-echo', + ), + fakeToolCall( + 'run_shell_command', + { command: 'ls -la' }, + 'shell-ls', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Run "echo hello" and "ls -la" commands.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Block all shell commands - excludeTools blocks entire tools @@ -314,6 +447,7 @@ describe('Tool Control Parameters (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -324,11 +458,30 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test content'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: '', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Clear the content of test.txt by writing empty string to it.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', // Conflicting settings: exclude takes priority @@ -361,6 +514,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toBe('test content'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -621,10 +775,34 @@ describe('Tool Control Parameters (E2E)', () => { let canUseToolCalled = false; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and write "modified" to it.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', coreTools: ['read_file', 'write_file'], @@ -661,6 +839,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).not.toBe('original'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -732,10 +911,34 @@ describe('Tool Control Parameters (E2E)', () => { const canUseToolCalls: string[] = []; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'test\n', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and append an empty line to it.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', // Only allow read_file, list_directory should trigger canUseTool @@ -772,6 +975,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(canUseToolCalls).not.toContain('read_file'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -784,10 +988,35 @@ describe('Tool Control Parameters (E2E)', () => { const canUseToolCalls: string[] = []; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'new', + }, + 'write-test', + ), + fakeToolCall('list_directory', { path: testDir }, 'list-dir'), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt, write "new" to it, and list the directory.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'auto-edit', // Allow list_directory in addition to auto-approved edit tools @@ -823,6 +1052,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(canUseToolCalls.length).toBe(0); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -985,10 +1215,39 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + fakeToolCall( + 'run_shell_command', + { command: 'echo hello' }, + 'shell-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and write "modified" to it.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', // Limit to specific tools @@ -1013,8 +1272,18 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); expect(toolNames).toContain('write_file'); - // Should NOT use tools outside coreTools - expect(toolNames).not.toContain('run_shell_command'); + const advertisedTools = advertisedToolNames(fakeServer); + expect(advertisedTools).toEqual( + expect.arrayContaining(['read_file', 'write_file']), + ); + + const shellResults = findToolResults(messages, 'run_shell_command'); + expect(shellResults).toHaveLength(1); + expect(shellResults[0]).toMatchObject({ + isError: true, + content: expect.stringContaining('was declined'), + }); + expect(advertisedTools).not.toContain('run_shell_command'); // Verify file was actually modified (content changed from original). // Don't assert on specific wording — the model may paraphrase. @@ -1022,6 +1291,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).not.toBe('test'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1032,11 +1302,37 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall('list_directory', { path: testDir }, 'list-dir'), + fakeToolCall( + 'edit', + { + file_path: helper.getPath('test.txt'), + old_string: 'test', + new_string: 'modified', + }, + 'edit-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt, write "new content" to it, and list directory.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Allow file operations @@ -1060,13 +1356,24 @@ describe('Tool Control Parameters (E2E)', () => { // Should use non-excluded tools from coreTools expect(toolNames).toContain('read_file'); - // Should NOT use excluded tool - expect(toolNames).not.toContain('edit'); + const advertisedTools = advertisedToolNames(fakeServer); + expect(advertisedTools).toEqual( + expect.arrayContaining(['read_file', 'list_directory']), + ); + + const editResults = findToolResults(messages, 'edit'); + expect(editResults).toHaveLength(1); + expect(editResults[0]).toMatchObject({ + isError: true, + content: expect.stringContaining('was declined'), + }); + expect(advertisedTools).not.toContain('edit'); // File should still exist expect(helper.fileExists('test.txt')).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1079,11 +1386,44 @@ describe('Tool Control Parameters (E2E)', () => { const canUseToolCalls: string[] = []; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + fakeToolCall( + 'edit', + { + file_path: helper.getPath('test.txt'), + old_string: 'test', + new_string: 'modified', + }, + 'edit-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt, write "modified" to it, and list the directory.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', // Limit available tools @@ -1113,8 +1453,22 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); expect(toolNames).toContain('write_file'); - // Should NOT use excluded tool - expect(toolNames).not.toContain('edit'); + const advertisedTools = advertisedToolNames(fakeServer); + expect(advertisedTools).toEqual( + expect.arrayContaining([ + 'read_file', + 'write_file', + 'list_directory', + ]), + ); + + const editResults = findToolResults(messages, 'edit'); + expect(editResults).toHaveLength(1); + expect(editResults[0]).toMatchObject({ + isError: true, + content: expect.stringContaining('was declined'), + }); + expect(advertisedTools).not.toContain('edit'); // canUseTool should be called for core write tools expect(canUseToolCalls).toContain('write_file'); @@ -1125,6 +1479,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).not.toBe('test'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1137,10 +1492,26 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Non-existent tool names should be ignored @@ -1163,6 +1534,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1173,10 +1545,26 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'test'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Non-existent tool names should be ignored @@ -1199,6 +1587,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1282,10 +1671,31 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'original'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'run_shell_command', + { command: 'echo hello' }, + 'shell-echo', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and run "echo hello" command.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // Even in yolo mode, excludeTools should block tools @@ -1318,6 +1728,7 @@ describe('Tool Control Parameters (E2E)', () => { } } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1332,11 +1743,30 @@ describe('Tool Control Parameters (E2E)', () => { const scenarioDir = await helper.mkdir(scenarioDirName); let capturedInput: Record = {}; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath(scenarioDirName + '/test.txt'), + content: 'new content', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Create a new file named test.txt with exactly this content: new content. Use the write_file tool.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: scenarioDir, permissionMode: 'default', coreTools: ['write_file'], @@ -1371,6 +1801,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toBe('new content'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1383,11 +1814,30 @@ describe('Tool Control Parameters (E2E)', () => { const scenarioDir = await helper.mkdir(scenarioDirName); let canUseToolCalled = false; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath(scenarioDirName + '/test.txt'), + content: 'modified', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Create a new file named test.txt with exactly this content: modified. Use the write_file tool.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: scenarioDir, permissionMode: 'default', coreTools: ['write_file'], @@ -1419,6 +1869,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toBe('modified'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1431,10 +1882,31 @@ describe('Tool Control Parameters (E2E)', () => { async () => { await helper.createFile('test.txt', 'original'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'edit', + { + file_path: helper.getPath('test.txt'), + old_string: 'original', + new_string: 'edited', + }, + 'edit-test', + ), + fakeToolCall('list_directory', { path: testDir }, 'list-dir'), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Edit test.txt and list the directory.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // edit is in coreTools but also in excludeTools @@ -1458,6 +1930,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('list_directory'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1469,10 +1942,34 @@ describe('Tool Control Parameters (E2E)', () => { await helper.createFile('test.txt', 'original'); await helper.createFile('other.txt', 'other content'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read test.txt and write "modified" to test.txt.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // write_file is in allowedTools but NOT in coreTools @@ -1499,6 +1996,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(toolNames).toContain('read_file'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1510,10 +2008,31 @@ describe('Tool Control Parameters (E2E)', () => { await helper.createFile('a.txt', 'content a'); await helper.createFile('b.txt', 'content b'); + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('a.txt') }, + 'read-a', + ), + fakeToolCall( + 'read_file', + { file_path: helper.getPath('b.txt') }, + 'read-b', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + const q = query({ prompt: 'Read both a.txt and b.txt files.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', // coreTools is the whitelist - only these tools can be used @@ -1542,6 +2061,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(uniqueTools).toEqual(['read_file']); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1560,6 +2080,40 @@ describe('Tool Control Parameters (E2E)', () => { input: Record; }> = []; + let streamingRequestIndex = 0; + const fakeServer = await startFakeOpenAIServer(({ body }) => { + if (body['stream'] !== true) { + return { content: '{"selected_memories":[]}' }; + } + const requestIndex = streamingRequestIndex++; + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + ], + }; + } + if (requestIndex === 1) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'updated', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + // Create an async generator that yields a single message async function* createPrompt(): AsyncIterable { yield { @@ -1579,6 +2133,7 @@ describe('Tool Control Parameters (E2E)', () => { prompt: createPrompt(), options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', coreTools: ['read_file', 'write_file'], @@ -1637,6 +2192,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(writtenContent).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1648,6 +2204,41 @@ describe('Tool Control Parameters (E2E)', () => { await helper.createFile('test.txt', INITIAL_CONTENT); const resultWaiter = createResultWaiter(1); + + let streamingRequestIndex = 0; + const fakeServer = await startFakeOpenAIServer(({ body }) => { + if (body['stream'] !== true) { + return { content: '{"selected_memories":[]}' }; + } + const requestIndex = streamingRequestIndex++; + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('test.txt') }, + 'read-test', + ), + ], + }; + } + if (requestIndex === 1) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('test.txt'), + content: 'modified', + }, + 'write-test', + ), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); + // Create an async generator that yields a single message async function* createPrompt(): AsyncIterable { yield { @@ -1669,6 +2260,7 @@ describe('Tool Control Parameters (E2E)', () => { prompt: createPrompt(), options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', coreTools: ['read_file', 'write_file'], @@ -1720,6 +2312,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toBe(INITIAL_CONTENT); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1733,6 +2326,42 @@ describe('Tool Control Parameters (E2E)', () => { const resultWaiter = createResultWaiter(2); const canUseToolCalls: string[] = []; + const fakeServer = await startFakeOpenAIServer(({ body }) => { + if (body['stream'] !== true) { + return { content: '{"selected_memories":[]}' }; + } + const transcript = JSON.stringify(body['messages'] ?? []); + if (transcript.includes('write-data')) { + return { content: 'Done.' }; + } + if (transcript.includes('Now append')) { + return { + toolCalls: [ + fakeToolCall( + 'write_file', + { + file_path: helper.getPath('data.txt'), + content: 'initial data - updated', + }, + 'write-data', + ), + ], + }; + } + if (transcript.includes('read-data')) { + return { content: 'Done.' }; + } + return { + toolCalls: [ + fakeToolCall( + 'read_file', + { file_path: helper.getPath('data.txt') }, + 'read-data', + ), + ], + }; + }, FAKE_SERVER_OPTIONS); + // Create an async generator that yields multiple messages async function* createMultiTurnPrompt(): AsyncIterable { const sessionId = crypto.randomUUID(); @@ -1766,6 +2395,7 @@ describe('Tool Control Parameters (E2E)', () => { prompt: createMultiTurnPrompt(), options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', coreTools: ['read_file', 'write_file'], @@ -1807,6 +2437,7 @@ describe('Tool Control Parameters (E2E)', () => { expect(content).toContain(' - updated'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT,