diff --git a/integration-tests/cli/acp-cron.test.ts b/integration-tests/cli/acp-cron.test.ts index 8eacb2bc3c8..3209f8e53f2 100644 --- a/integration-tests/cli/acp-cron.test.ts +++ b/integration-tests/cli/acp-cron.test.ts @@ -11,18 +11,23 @@ * and stream results back to the client via sessionUpdate notifications, * even after the originating prompt has already returned. * - * The two tests share one ACP session to stay within 2 minutes total: - * 1. Fast smoke test — cron tools available (no cron fire needed) - * 2. Combined test — create job, verify session responsive, wait for - * cron fire, check content + _meta.source, then clean up + * Uses fake-openai-server for deterministic model responses, eliminating + * model output variance as a failure source. The cron scheduler still + * operates on real minute-boundary timing. */ import { spawn } from 'node:child_process'; -import { readFileSync, writeFileSync } from 'node:fs'; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { basename, dirname, join } from 'node:path'; import { createInterface } from 'node:readline'; import { setTimeout as delay } from 'node:timers/promises'; import { describe, it, expect } from 'vitest'; import { TestRig } from '../test-helper.js'; +import { + startFakeOpenAIServer, + fakeToolCall, + type FakeOpenAIServer, +} from '../fake-openai-server.js'; const REQUEST_TIMEOUT_MS = 60_000; @@ -69,9 +74,10 @@ type PermissionRequest = { }; /** - * Sets up an ACP test environment with cron support enabled. + * Sets up an ACP test environment with cron support enabled, backed by + * a fake-openai-server for deterministic model responses. */ -function setupAcpCronTest(rig: TestRig) { +function setupAcpCronTest(rig: TestRig, fakeServer: FakeOpenAIServer) { const pending = new Map(); let nextRequestId = 1; const sessionUpdates: (SessionUpdateNotification & { @@ -79,6 +85,15 @@ function setupAcpCronTest(rig: TestRig) { })[] = []; const stderr: string[] = []; + // Keep the global config dir outside the agent's workspace cwd so workspace + // scans (memory discovery, file tooling) never see it. + const qwenHome = join( + dirname(rig.testDir!), + `${basename(rig.testDir!)}-home`, + ); + rmSync(qwenHome, { recursive: true, force: true }); + mkdirSync(qwenHome, { recursive: true }); + const agent = spawn( 'node', [rig.bundlePath, '--acp', '--no-chat-recording'], @@ -87,6 +102,14 @@ function setupAcpCronTest(rig: TestRig) { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, + QWEN_HOME: qwenHome, + OPENAI_API_KEY: 'fake-key', + OPENAI_BASE_URL: fakeServer.baseUrl, + OPENAI_MODEL: 'fake-model', + QWEN_MODEL: 'fake-model', + // Defends against an ambient proxy intercepting the local fake server. + NO_PROXY: '127.0.0.1,localhost', + no_proxy: '127.0.0.1,localhost', }, }, ); @@ -263,6 +286,7 @@ function setupAcpCronTest(rig: TestRig) { pending.forEach(({ timeout }) => clearTimeout(timeout)); pending.clear(); await waitForExit(); + rmSync(qwenHome, { recursive: true, force: true }); }; return { @@ -303,86 +327,112 @@ async function initSession( const rig = new TestRig(); rig.setup('acp-cron-e2e'); - const { - sendRequest, - cleanup, - stderr, - // sessionUpdates available for debugging - waitForSessionUpdate, - } = setupAcpCronTest(rig); + // Only requestIndex 0 is load-bearing: it returns the cron_create + // tool call. The CLI makes internal model calls (tool-call + // classification, suggestion mode) between user-facing turns, so + // later indices do not map 1:1 to the prompts sent below. No + // assertion reads scripted response content, so the default reply + // suffices for every other turn. + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall('cron_create', { + cron: '*/1 * * * *', + prompt: 'Say CRONFIRE7742 and nothing else', + recurring: true, + }), + ], + }; + } + return { content: 'Done.' }; + }); try { - const sessionId = await initSession(sendRequest, rig.testDir!); - - // --- Part 1: Create a cron job that fires every minute --- - const createResult = (await sendRequest('session/prompt', { - sessionId, - prompt: [ - { - type: 'text', - text: 'Call cron_create with cron expression "*/1 * * * *" and prompt "Say CRONFIRE7742 and nothing else" and recurring true. Confirm briefly.', - }, - ], - })) as { stopReason: string }; - expect(createResult.stopReason).toBe('end_turn'); - - const promptDoneAt = Date.now(); - - // --- Part 2: Session stays responsive while cron is pending --- - const interactiveResult = (await sendRequest('session/prompt', { - sessionId, - prompt: [ - { - type: 'text', - text: 'Say INTERACTIVE8899 and nothing else.', - }, - ], - })) as { stopReason: string }; - expect(interactiveResult.stopReason).toBe('end_turn'); - - // --- Part 3: Wait for cron-fired notification (up to 75s) --- - // The cron fires at the next minute boundary. The model response - // should stream back as sessionUpdate notifications after the - // originating prompt has already returned. - - // 3a: Check for user_message_chunk echoing the cron prompt with _meta.source - const cronUserMsg = await waitForSessionUpdate( - (u) => - u.update?.sessionUpdate === 'user_message_chunk' && - (u.update?.content?.text ?? '').includes('CRONFIRE7742') && - u.receivedAt > promptDoneAt, - 'cron-fired user_message_chunk with CRONFIRE7742', - 75_000, - ); - expect(cronUserMsg.update?._meta).toBeDefined(); - expect(cronUserMsg.update?._meta?.source).toBe('cron'); - - // 3b: Check for agent_message_chunk after the cron user message - // (the model's response to the cron prompt) - const cronAgentMsg = await waitForSessionUpdate( - (u) => - u.update?.sessionUpdate === 'agent_message_chunk' && - u.receivedAt > cronUserMsg.receivedAt, - 'agent_message_chunk after cron fire', - 15_000, // should already be here by now - ); - expect(cronAgentMsg.receivedAt).toBeGreaterThan(promptDoneAt); - - // --- Part 4: Clean up the cron job --- - await sendRequest('session/prompt', { - sessionId, - prompt: [ - { - type: 'text', - text: 'Delete all cron jobs using cron_delete.', - }, - ], - }); - } catch (e) { - if (stderr.length) console.error('Agent stderr:', stderr.join('')); - throw e; + const { sendRequest, cleanup, stderr, waitForSessionUpdate } = + setupAcpCronTest(rig, fakeServer); + + try { + const sessionId = await initSession(sendRequest, rig.testDir!); + + // --- Part 1: Create a cron job that fires every minute --- + const createResult = (await sendRequest('session/prompt', { + sessionId, + prompt: [ + { + type: 'text', + text: 'Call cron_create with cron expression "*/1 * * * *" and prompt "Say CRONFIRE7742 and nothing else" and recurring true. Confirm briefly.', + }, + ], + })) as { stopReason: string }; + expect(createResult.stopReason).toBe('end_turn'); + + // Fail fast if the cron_create tool call was not served to the first + // user prompt. An internal model call before the first prompt (title + // generation, a classifier pass) would shift dispatch and otherwise + // surface only as an opaque 75s timeout in Part 3. + expect( + JSON.stringify(fakeServer.requests[0]?.body['messages']), + 'requestIndex 0 was not the cron_create prompt — dispatch shifted', + ).toContain('CRONFIRE7742'); + + const promptDoneAt = Date.now(); + + // --- Part 2: Session stays responsive while cron is pending --- + const interactiveResult = (await sendRequest('session/prompt', { + sessionId, + prompt: [ + { + type: 'text', + text: 'Say INTERACTIVE8899 and nothing else.', + }, + ], + })) as { stopReason: string }; + expect(interactiveResult.stopReason).toBe('end_turn'); + + // --- Part 3: Wait for cron-fired notification (up to 75s) --- + // The cron fires at the next minute boundary. The model response + // should stream back as sessionUpdate notifications after the + // originating prompt has already returned. + + // 3a: Check for user_message_chunk echoing the cron prompt with _meta.source + const cronUserMsg = await waitForSessionUpdate( + (u) => + u.update?.sessionUpdate === 'user_message_chunk' && + (u.update?.content?.text ?? '').includes('CRONFIRE7742') && + u.receivedAt > promptDoneAt, + 'cron-fired user_message_chunk with CRONFIRE7742', + 75_000, + ); + expect(cronUserMsg.update?._meta).toBeDefined(); + expect(cronUserMsg.update?._meta?.source).toBe('cron'); + + // 3b: Check for agent_message_chunk after the cron user message + // (the model's response to the cron prompt) + const cronAgentMsg = await waitForSessionUpdate( + (u) => + u.update?.sessionUpdate === 'agent_message_chunk' && + u.receivedAt > cronUserMsg.receivedAt, + 'agent_message_chunk after cron fire', + 15_000, // should already be here by now + ); + expect(cronAgentMsg.receivedAt).toBeGreaterThan(promptDoneAt); + } catch (e) { + if (stderr.length) console.error('Agent stderr:', stderr.join('')); + console.error( + 'Fake server requests:', + fakeServer.requests.map((r, i) => ({ + index: i, + model: r.body['model'], + messages: JSON.stringify(r.body['messages']).slice(0, 300), + })), + ); + throw e; + } finally { + await cleanup(); + } } finally { - await cleanup(); + await fakeServer.close(); } }, { timeout: 120_000, retry: 0 },