diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e0e785fa3b..5af4b8c650 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -135,6 +135,34 @@ jobs: - name: Run Bun runtime suite run: deno task test:bun + tests-runtime-critical-flow: + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 20 + name: "tests (runtime critical flow: ${{ matrix.runtime }})" + strategy: + fail-fast: false + matrix: + runtime: [deno, node, bun] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-deno + timeout-minutes: 5 + with: + warm-cache: "true" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 + with: + node-version: "24" + package-manager-cache: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + if: ${{ matrix.runtime == 'bun' }} + with: + bun-version: "1.3.6" + - name: Run runtime critical flow + run: deno task test:e2e:runtime-inference-critical-flow --runtime=${{ matrix.runtime }} + coverage-shards: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest diff --git a/deno.json b/deno.json index 5850e65313..41d6dede96 100644 --- a/deno.json +++ b/deno.json @@ -559,7 +559,7 @@ "lint:anti-slop": "deno run --allow-read scripts/lint/audit-anti-slop.ts", "lint:render-mode-defaults": "deno run --allow-read scripts/lint/audit-render-mode-defaults.ts", "lint:dnt-meta-properties": "deno run --config=scripts/test.deno.json --frozen --allow-read scripts/build/dnt-meta-property-safety.ts", - "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-rfc-status.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts && deno task test:tool-search-live", + "test:scripts": "deno test --config=scripts/test.deno.json --no-check --allow-read --allow-write --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE --allow-run scripts/lint/check-deno-version.test.ts scripts/ci/prepare-rc-build.test.ts scripts/ci/publish-npm-packages.test.ts scripts/ci/setup-deno-workflow.test.ts scripts/test/coverage-ci.test.ts scripts/test/run-test-file.test.ts scripts/build/compile-binary.test.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.test.ts scripts/build/generate-sbom.test.ts scripts/build/generated-artifact-checks.test.ts scripts/build/npm-dependency-sources.test.ts scripts/build/npm-extension-package-metadata.test.ts scripts/build/npm-package-metadata.test.ts scripts/build/npm-react-shims.test.ts scripts/build/npm-runtime-helper-contract.test.ts scripts/build/prepare-framework-sources.test.ts scripts/build/report-artifact-sizes.test.ts scripts/build/run-generate.test.ts scripts/docs/docs-coverage.test.ts scripts/docs/generate-api-reference.test.ts scripts/docs/guide-validation.test.ts scripts/docs/validate-tracked-docs.test.ts scripts/lint/audit-chat-composability.test.ts scripts/lint/audit-rfc-status.test.ts scripts/lint/audit-core-deps.test.ts scripts/lint/audit-cwd-relative-test-reads.test.ts scripts/lint/audit-cross-runtime-jsr.test.ts scripts/lint/audit-dependency-boundaries.test.ts scripts/lint/audit-extension-capabilities.test.ts scripts/lint/audit-extension-contracts.test.ts scripts/lint/audit-deps.test.ts scripts/lint/check-module-boundaries.test.ts scripts/lint/lint-config.test.ts scripts/lint/ban-test-only.test.ts scripts/lint/check-sanitizer-baseline.test.ts scripts/lint/check-skipped-tests-baseline.test.ts scripts/lint/audit-anti-slop.test.ts scripts/lint/audit-render-mode-defaults.test.ts scripts/lint/check-test-typecheck-baseline.test.ts scripts/lint/check-coverage.test.ts scripts/security/audit-npm.test.ts scripts/security/secret-scanning-config.test.ts scripts/security/submit-dependency-snapshot.test.ts scripts/test/template-runtime-e2e.test.ts scripts/test/runtime-inference-critical-flow.test.ts && deno task test:tool-search-live", "test:sentry-runtime-packages": "deno test --config=scripts/test.deno.json --no-check --no-lock --allow-read --allow-write --allow-run --allow-env=DENO_DIR,HOME,XDG_CACHE_HOME,LOCALAPPDATA,USERPROFILE scripts/build/sentry-runtime-packages.test.ts", "test:tool-search-live": "VF_DISABLE_LRU_INTERVAL=1 deno test --no-check -A tests/agent/verify-tool-search-live.test.ts", "test:cross-runtime": "deno run --allow-all src/platform/compat/cross-runtime.test.ts", @@ -578,6 +578,7 @@ "test:e2e:binary": "deno task generate && deno test --allow-all tests/integration/compiled-binary-e2e.test.ts", "test:e2e:binary:fresh": "deno task generate && VERYFRONT_BINARY_FRESH=1 deno test --allow-all tests/integration/compiled-binary-e2e.test.ts", "test:e2e:templates": "deno run --allow-all scripts/test/template-runtime-e2e.ts", + "test:e2e:runtime-inference-critical-flow": "deno run --allow-all scripts/test/runtime-inference-critical-flow.ts", "check:circular": "deno task lint:module-boundaries", "cli": "deno run --allow-all cli/main.ts", "mcp": "deno run --allow-all cli/main.ts mcp", diff --git a/scripts/test/runtime-e2e-helpers.ts b/scripts/test/runtime-e2e-helpers.ts new file mode 100644 index 0000000000..2869f82164 --- /dev/null +++ b/scripts/test/runtime-e2e-helpers.ts @@ -0,0 +1,410 @@ +import { + type CommandResult as ManagedCommandResult, + runCommand as runManagedCommand, +} from "../../src/platform/compat/process/command.ts"; + +export type RuntimeName = "node" | "bun" | "deno"; + +export interface CommandResult { + code: number; + stdout: string; + stderr: string; +} + +const decoder = new TextDecoder(); + +export function parseCommaSeparatedFlag( + args: string[], + names: string[], +): string[] | null { + for (const name of names) { + const prefix = `--${name}=`; + const inline = args.find((arg) => arg.startsWith(prefix)); + if (inline) { + return inline.slice(prefix.length).split(",").map((value) => value.trim()) + .filter(Boolean); + } + + const index = args.indexOf(`--${name}`); + if (index >= 0) { + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`--${name} requires a comma-separated value`); + } + return value.split(",").map((entry) => entry.trim()).filter(Boolean); + } + } + + return null; +} + +async function runCommand( + command: string, + args: string[], + options: { + cwd?: string; + env?: Record; + timeoutMs?: number; + } = {}, +): Promise { + const controller = new AbortController(); + const timeout = options.timeoutMs === undefined + ? undefined + : setTimeout(() => controller.abort(), options.timeoutMs); + + try { + const output = await new Deno.Command(command, { + args, + cwd: options.cwd, + env: options.env, + signal: controller.signal, + stdout: "piped", + stderr: "piped", + }).output(); + + return { + code: output.code, + stdout: decoder.decode(output.stdout), + stderr: decoder.decode(output.stderr), + }; + } catch (error) { + if (controller.signal.aborted) { + throw new Error( + `${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`, + ); + } + throw error; + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +export async function runChecked( + command: string, + args: string[], + options: { + cwd?: string; + env?: Record; + timeoutMs?: number; + } = {}, +): Promise { + const result = await runCommand(command, args, options); + if (result.code !== 0) { + throw new Error( + [ + `${command} ${args.join(" ")} failed with exit code ${result.code}`, + result.stdout.trim(), + result.stderr.trim(), + ].filter(Boolean).join("\n"), + ); + } + return result; +} + +export async function ensureCommand( + command: string, + args: string[] = ["--version"], +): Promise { + await runChecked(command, args, { timeoutMs: 30_000 }); +} + +export async function inspectModuleExports( + moduleUrl: URL, + label: string, + timeoutMs = 7_500, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let result: Deno.CommandOutput; + + try { + result = await new Deno.Command(Deno.execPath(), { + args: [ + "eval", + `--config=${new URL("../test.deno.json", import.meta.url).pathname}`, + "--no-check", + `const mod = await import(${JSON.stringify(moduleUrl.href)});\n` + + "console.log(JSON.stringify(Object.keys(mod).sort()));", + ], + signal: controller.signal, + stdout: "piped", + stderr: "piped", + }).output(); + } catch (error) { + if (controller.signal.aborted) { + throw new Error( + `${label} import subprocess timed out after ${timeoutMs}ms`, + ); + } + throw error; + } finally { + clearTimeout(timeout); + } + + const stderr = decoder.decode(result.stderr); + if (stderr.length > 0) { + throw new Error(`${label} import subprocess wrote to stderr:\n${stderr}`); + } + if (result.code !== 0) { + throw new Error( + `${label} import subprocess exited with code ${result.code}`, + ); + } + + return JSON.parse(decoder.decode(result.stdout)) as string[]; +} + +export async function packNpmPackage( + rootDir: string, + workDir: string, +): Promise { + const packDir = `${workDir}/packed`; + await Deno.mkdir(packDir, { recursive: true }); + const result = await runChecked("npm", [ + "pack", + "--pack-destination", + packDir, + ], { + cwd: `${rootDir}/npm`, + timeoutMs: 120_000, + }); + const tarball = result.stdout.split(/\r?\n/) + .map((line) => line.trim()) + .findLast((line) => line.endsWith(".tgz")); + + if (!tarball) { + throw new Error(`npm pack did not report a tarball:\n${result.stdout}`); + } + + return `${packDir}/${tarball}`; +} + +async function updateVeryfrontDependency( + projectDir: string, + tarballPath: string, +): Promise { + const packagePath = `${projectDir}/package.json`; + const pkg = JSON.parse(await Deno.readTextFile(packagePath)); + pkg.dependencies ??= {}; + pkg.dependencies.veryfront = `file:${tarballPath}`; + await Deno.writeTextFile(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); +} + +async function usePackedVeryfrontDenoTasks( + projectDir: string, + tarballPath: string, +): Promise { + const packagePath = `${projectDir}/package.json`; + const pkg = JSON.parse(await Deno.readTextFile(packagePath)); + delete pkg.dependencies?.veryfront; + await Deno.writeTextFile(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); + + const packedCliDir = `${projectDir}/.veryfront-packed-cli`; + await Deno.mkdir(packedCliDir, { recursive: true }); + await runChecked("tar", ["-xzf", tarballPath, "-C", packedCliDir], { + timeoutMs: 30_000, + }); + await runChecked("deno", ["install"], { + cwd: `${packedCliDir}/package`, + timeoutMs: 180_000, + }); + + const cliPath = JSON.stringify(`${packedCliDir}/package/esm/cli/main.js`); + const denoConfigPath = `${projectDir}/deno.json`; + const config = JSON.parse(await Deno.readTextFile(denoConfigPath)); + config.tasks ??= {}; + config.tasks.dev = `deno run -A ${cliPath} dev`; + config.tasks.build = `deno run -A ${cliPath} build`; + config.tasks.preview = `deno run -A ${cliPath} preview`; + await Deno.writeTextFile( + denoConfigPath, + `${JSON.stringify(config, null, 2)}\n`, + ); +} + +export function assertCondition(condition: boolean, message: string): void { + if (!condition) throw new Error(message); +} + +export function allocatePort(): number { + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + const port = (listener.addr as Deno.NetAddr).port; + listener.close(); + return port; +} + +export async function waitForRoute( + url: string, + timeoutMs = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastError = ""; + + while (Date.now() < deadline) { + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 1_000); + try { + const response = await fetch(url, { signal: controller.signal }); + if (response.ok) { + await response.body?.cancel(); + return; + } + lastError = `HTTP ${response.status}`; + await response.body?.cancel(); + } finally { + clearTimeout(timeout); + } + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + } + + throw new Error( + `${url} did not become ready within ${timeoutMs}ms: ${lastError}`, + ); +} + +export function getDevServerCommand( + runtime: RuntimeName, + port: number, +): { command: string; args: string[] } { + return { + command: runtime === "node" ? "npm" : runtime, + args: runtime === "deno" + ? ["task", "dev", "--port", String(port)] + : ["run", "dev", "--", "--port", String(port)], + }; +} + +export function getDevServerEnvironment( + overrides: Record = {}, +): Record { + return { + ANTHROPIC_API_KEY: "", + GOOGLE_API_KEY: "", + GOOGLE_GENERATIVE_AI_API_KEY: "", + LOG_FORMAT: "text", + MISTRAL_API_KEY: "", + NODE_ENV: "development", + OPENAI_API_KEY: "", + REVALIDATION_PER_PROJECT_LIMIT: "0", + SSR_TRANSFORM_PER_PROJECT_LIMIT: "0", + VERYFRONT_API_TOKEN: "", + VF_DISABLE_LRU_INTERVAL: "1", + ...overrides, + }; +} + +export function startDevServer( + projectDir: string, + runtime: RuntimeName, + port: number, + env: Record = {}, +): { + abortController: AbortController; + result: Promise; + stdout: string[]; + stderr: string[]; +} { + const { command, args } = getDevServerCommand(runtime, port); + const stdout: string[] = []; + const stderr: string[] = []; + const abortController = new AbortController(); + const result = runManagedCommand(command, { + args, + cwd: projectDir, + env: getDevServerEnvironment(env), + capture: true, + signal: abortController.signal, + terminateProcessTreeOnExit: true, + }).then((commandResult) => { + if (commandResult.stdout) stdout.push(commandResult.stdout); + if (commandResult.stderr) stderr.push(commandResult.stderr); + return commandResult; + }); + + return { abortController, result, stdout, stderr }; +} + +export async function stopDevServer(server: { + abortController: AbortController; + result: Promise; +}): Promise { + server.abortController.abort(); + await server.result; +} + +export async function scaffoldProject( + workDir: string, + tarballPath: string, + template: string, + runtime: RuntimeName, +): Promise { + const caseDir = `${workDir}/${runtime}-${template}`; + const projectName = `vf-${runtime}-${template}`; + await Deno.mkdir(caseDir, { recursive: true }); + await runChecked("npm", [ + "exec", + "--yes", + "--package", + tarballPath, + "--", + "veryfront", + "init", + projectName, + "--template", + template, + "--runtime", + runtime, + "--skip-install", + "--skip-env-prompt", + ], { + cwd: caseDir, + env: { + npm_config_cache: `${workDir}/npm-cache`, + npm_config_fund: "false", + npm_config_audit: "false", + }, + timeoutMs: 120_000, + }); + + const projectDir = `${caseDir}/${projectName}`; + if (runtime === "deno") { + await usePackedVeryfrontDenoTasks(projectDir, tarballPath); + } else { + await updateVeryfrontDependency(projectDir, tarballPath); + } + + return projectDir; +} + +export async function installDependencies( + projectDir: string, + runtime: RuntimeName, + workDir: string, +): Promise { + if (runtime === "node") { + await runChecked("npm", ["install", "--no-audit", "--fund=false"], { + cwd: projectDir, + env: { npm_config_cache: `${workDir}/npm-cache` }, + timeoutMs: 180_000, + }); + return; + } + + if (runtime === "deno") { + await runChecked("deno", ["install"], { + cwd: projectDir, + timeoutMs: 180_000, + }); + return; + } + + await runChecked("bun", ["install"], { + cwd: projectDir, + timeoutMs: 180_000, + }); +} diff --git a/scripts/test/runtime-inference-critical-flow.test.ts b/scripts/test/runtime-inference-critical-flow.test.ts new file mode 100644 index 0000000000..ce244a6768 --- /dev/null +++ b/scripts/test/runtime-inference-critical-flow.test.ts @@ -0,0 +1,801 @@ +import { + assert, + assertEquals, + assertRejects, + assertStringIncludes, + assertThrows, +} from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { parse } from "#std/yaml/parse"; +import { + artifactClaim, + assertListedRunFailure, + parseAgUiTextDeltas, + parseRuntimeSelection, + parseScopedResponseJson, + validateAnthropicRequest, + waitForProviderCancellation, + waitForProviderReceipt, + waitForTerminalRun, +} from "./runtime-inference-critical-flow.ts"; +import { + inspectModuleExports, + parseCommaSeparatedFlag, +} from "./runtime-e2e-helpers.ts"; +import { withMockFetch } from "#veryfront/testing/mock-fetch.ts"; + +const VALID_WIRE_MODEL = "claude-haiku-4-5-20251001"; +const VALID_KEY = "vf-runtime-critical-flow-key"; +const FLOW_TEST_PATH = "scripts/test/runtime-inference-critical-flow.test.ts"; +const FLOW_HARNESS_PATH = "scripts/test/runtime-inference-critical-flow.ts"; +const REPO_ROOT = new URL("../../", import.meta.url); + +function anthropicRequest(overrides: { + method?: string; + path?: string; + key?: string; + anthropicVersion?: string | null; + contentType?: string | null; + body?: unknown; +} = {}): Request { + const headers = new Headers({ + "x-api-key": overrides.key ?? VALID_KEY, + }); + if (overrides.anthropicVersion !== null) { + headers.set( + "anthropic-version", + overrides.anthropicVersion ?? "2023-06-01", + ); + } + if (overrides.contentType !== null) { + headers.set( + "content-type", + overrides.contentType ?? "application/json", + ); + } + return new Request(`http://127.0.0.1${overrides.path ?? "/v1/messages"}`, { + method: overrides.method ?? "POST", + headers, + body: JSON.stringify( + overrides.body ?? { + model: VALID_WIRE_MODEL, + messages: [{ role: "user", content: "marker: contract-marker" }], + }, + ), + }); +} + +async function assertRejectsWithMessage( + fn: () => Promise, + expected: string, + assertionMessage: string, +): Promise { + const error = await assertRejects( + fn, + Error, + undefined, + assertionMessage, + ); + assertStringIncludes((error as Error).message, expected, assertionMessage); +} + +function yamlRecord(value: unknown, context: string): Record { + assert( + value !== null && typeof value === "object" && !Array.isArray(value), + `${context} must be an object`, + ); + return value as Record; +} + +function unresolvedCancellationFields(): { + cancellationEvidence: () => undefined; + cancellation: Promise; +} { + return { + cancellationEvidence: () => undefined, + cancellation: new Promise(() => {}), + }; +} + +describe("runtime inference critical-flow pure contract", () => { + it("selects all runtimes by default in stable order", () => { + assertEquals( + parseRuntimeSelection([]), + ["node", "bun", "deno"], + "Default runtime selection should cover all supported runtimes in stable order", + ); + }); + + it("selects explicit runtimes from inline or separate flags", () => { + assertEquals( + parseRuntimeSelection(["--runtime=node"]), + ["node"], + "Inline --runtime should select exactly the requested runtime", + ); + assertEquals( + parseRuntimeSelection(["--runtime", "bun"]), + ["bun"], + "Separate --runtime value should select exactly the requested runtime", + ); + assertEquals( + parseRuntimeSelection(["--runtimes=deno,node"]), + ["deno", "node"], + "Comma-separated --runtimes should preserve explicit runtime order", + ); + assertEquals( + parseCommaSeparatedFlag(["--runtime=node,bun"], ["runtime"]), + ["node", "bun"], + "Shared comma flag parser should return trimmed runtime entries", + ); + }); + + it("rejects missing, unknown, and duplicate runtime selections", () => { + assertThrows( + () => parseRuntimeSelection(["--runtime"]), + Error, + "--runtime requires a comma-separated value", + "Missing runtime flag value should throw the parser's validation error", + ); + assertThrows( + () => parseRuntimeSelection(["--runtime=python"]), + Error, + "Unknown runtime: python", + "Unknown runtime names should be rejected before execution", + ); + assertThrows( + () => parseRuntimeSelection(["--runtime=node,node"]), + Error, + "Duplicate runtime: node", + "Duplicate runtime names should be rejected before execution", + ); + }); + + it("states honest runtime artifact claims", () => { + assertStringIncludes( + artifactClaim("node"), + "packed npm consumer", + "Node artifact claim should name npm consumer coverage", + ); + assertStringIncludes( + artifactClaim("bun"), + "packed package consumer", + "Bun artifact claim should name package consumer coverage", + ); + assertStringIncludes( + artifactClaim("deno"), + "packed CLI", + "Deno artifact claim should name packed CLI coverage", + ); + assertEquals( + artifactClaim("deno").includes("npm install"), + false, + "Deno artifact claim should not imply npm installation semantics", + ); + }); + + it("accepts exact Anthropic requests with the expected marker", async () => { + await validateAnthropicRequest( + anthropicRequest(), + "contract-marker", + ); + }); + + it("rejects invalid Anthropic request shape before recording success", async () => { + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + new Request("http://127.0.0.1/v1/messages", { method: "GET" }), + "contract-marker", + ), + "POST /v1/messages", + "Validator should reject requests that do not use POST /v1/messages", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ path: "/wrong" }), + "contract-marker", + ), + "POST /v1/messages", + "Validator should reject requests sent to the wrong Anthropic path", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ key: "wrong" }), + "contract-marker", + ), + "x-api-key", + "Validator should reject requests without the expected test API key", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ anthropicVersion: null }), + "contract-marker", + ), + "anthropic-version", + "Validator should reject requests without the required Anthropic version", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ contentType: "text/plain" }), + "contract-marker", + ), + "application/json", + "Validator should reject requests without a JSON content type", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + new Request("http://127.0.0.1/v1/messages", { + method: "POST", + headers: { + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "x-api-key": VALID_KEY, + }, + body: "{", + }), + "contract-marker", + ), + "valid JSON", + "Validator should preserve invalid JSON validation failures", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ body: { model: "wrong", messages: [] } }), + "contract-marker", + ), + VALID_WIRE_MODEL, + "Validator should preserve wrong model validation failures", + ); + await assertRejectsWithMessage( + () => + validateAnthropicRequest( + anthropicRequest({ body: { model: VALID_WIRE_MODEL, messages: [] } }), + "contract-marker", + ), + "contract-marker", + "Validator should reject requests that omit the workflow marker", + ); + }); + + it("extracts assistant text deltas without matching user-message snapshots", () => { + const marker = "agent-output-marker"; + const stream = [ + `event: MessagesSnapshot\ndata: ${ + JSON.stringify({ messages: [{ role: "user", content: marker }] }) + }`, + `event: TextMessageContent\ndata: ${JSON.stringify({ delta: marker })}`, + `event: RunFinished\ndata: {}`, + "", + ].join("\n\n"); + assertEquals( + parseAgUiTextDeltas(stream), + [marker], + "AG-UI output proof should read assistant TextMessageContent deltas only", + ); + assertEquals( + parseAgUiTextDeltas( + `event: MessagesSnapshot\ndata: ${ + JSON.stringify({ messages: [{ role: "user", content: marker }] }) + }\n\n`, + ), + [], + "A user-message snapshot must not count as assistant/provider output", + ); + }); + + it("surfaces provider validation failures instead of generic receipt timeouts", async () => { + const detailCalls: string[] = []; + await withMockFetch( + ((input: URL | RequestInfo) => { + detailCalls.push(String(input)); + return Promise.resolve(Response.json({ status: "running" })); + }) as typeof fetch, + async () => { + await assertRejectsWithMessage( + () => + waitForProviderReceipt( + { + received: [], + server: {} as Deno.HttpServer, + abort() {}, + closed: Promise.resolve(), + validationFailure: () => + new Error( + `Expected Anthropic model ${VALID_WIRE_MODEL}, got wrong`, + ), + ...unresolvedCancellationFields(), + url: new URL("http://127.0.0.1:1/v1"), + }, + "node", + new URL("http://127.0.0.1/runs/validation-failed"), + ), + `Expected Anthropic model ${VALID_WIRE_MODEL}, got wrong`, + "Provider receipt wait should surface validation failures before polling detail", + ); + assertEquals( + detailCalls.length, + 0, + "Provider receipt wait should not poll run detail after validation already failed", + ); + }, + ); + }); + + it("surfaces provider validation failures that arrive with terminal detail", async () => { + let validationFailure: Error | undefined; + await withMockFetch( + (() => { + validationFailure = new Error( + `Expected Anthropic model ${VALID_WIRE_MODEL}, got wrong`, + ); + return Promise.resolve(Response.json({ status: "failed" })); + }) as typeof fetch, + async () => { + await assertRejectsWithMessage( + () => + waitForProviderReceipt( + { + received: [], + server: {} as Deno.HttpServer, + abort() {}, + closed: Promise.resolve(), + validationFailure: () => validationFailure, + ...unresolvedCancellationFields(), + url: new URL("http://127.0.0.1:1/v1"), + }, + "node", + new URL("http://127.0.0.1/runs/terminal-validation-failed"), + ), + `Expected Anthropic model ${VALID_WIRE_MODEL}, got wrong`, + "Provider receipt wait should surface validation failures observed during terminal polling", + ); + }, + ); + }); + + it("preserves terminal-detail validation failure observed during provider wait", async () => { + const expectedMessage = + `Expected Anthropic model ${VALID_WIRE_MODEL}, got wrong`; + let validationFailure: Error | undefined; + await withMockFetch( + (() => { + validationFailure = new Error(expectedMessage); + return Promise.resolve(Response.json({ status: "failed" })); + }) as typeof fetch, + async () => { + const error = await assertRejects( + () => + waitForProviderReceipt( + { + received: [], + server: {} as Deno.HttpServer, + abort() {}, + closed: Promise.resolve(), + validationFailure: () => validationFailure, + ...unresolvedCancellationFields(), + url: new URL("http://127.0.0.1:1/v1"), + }, + "node", + new URL("http://127.0.0.1/runs/deadline-edge-validation-failed"), + ), + Error, + undefined, + "Provider wait should reject when validation fails during terminal polling", + ); + assertEquals( + (error as Error).message, + expectedMessage, + "Provider wait should not replace validation error text with generic timeout text", + ); + }, + ); + }); + + it("requires provider-side client abort evidence before cleanup can release the black-hole response", async () => { + await waitForProviderCancellation( + { + received: [], + server: {} as Deno.HttpServer, + abort() { + throw new Error("cleanup abort should not be needed for evidence"); + }, + closed: Promise.resolve(), + validationFailure: () => undefined, + cancellationEvidence: () => "client connection aborted", + cancellation: Promise.resolve("client connection aborted"), + url: new URL("http://127.0.0.1:1/v1"), + }, + "node", + 25, + ); + + await assertRejectsWithMessage( + () => + waitForProviderCancellation( + { + received: [], + server: {} as Deno.HttpServer, + abort() { + throw new Error( + "cleanup abort should not be needed for evidence", + ); + }, + closed: Promise.resolve(), + validationFailure: () => undefined, + cancellationEvidence: () => undefined, + cancellation: new Promise(() => {}), + url: new URL("http://127.0.0.1:1/v1"), + }, + "node", + 10, + ), + "provider request was not aborted by the client before cleanup", + "Provider cancellation wait should fail if cleanup would be the only release path", + ); + }); + + it("polls through non-terminal states and returns terminal failure details", async () => { + const states = [ + { status: "pending" }, + { status: "running" }, + { + status: "failed", + nodeStates: { "call-provider": { status: "failed" } }, + }, + ]; + let calls = 0; + await withMockFetch( + (() => { + const body = states[Math.min(calls, states.length - 1)]; + calls += 1; + return Promise.resolve(Response.json(body)); + }) as typeof fetch, + async () => { + const detail = await waitForTerminalRun( + new URL("http://127.0.0.1/runs/1"), + 1_000, + ); + assertEquals( + detail.status, + "failed", + "Terminal run polling should return the failed detail payload", + ); + assertEquals( + calls, + 3, + "Terminal run polling should continue through pending and running states", + ); + }, + ); + }); + + it("rejects terminal success and preserves last observation on deadline", async () => { + await withMockFetch( + (() => + Promise.resolve( + Response.json({ status: "completed" }), + )) as typeof fetch, + async () => { + await assertRejectsWithMessage( + () => waitForTerminalRun(new URL("http://127.0.0.1/runs/ok"), 1_000), + "unexpected terminal status", + "Terminal run polling should reject completed runs for the timeout contract", + ); + }, + ); + + await withMockFetch( + (() => + Promise.resolve( + Response.json({ + status: "running", + nodeStates: { "call-provider": { status: "running" } }, + }), + )) as typeof fetch, + async () => { + const error = await assertRejects( + () => waitForTerminalRun(new URL("http://127.0.0.1/runs/slow"), 40), + Error, + undefined, + "Terminal run polling should reject when the deadline expires", + ); + assertStringIncludes( + (error as Error).message, + "timed out", + "Terminal run polling deadline should report a timeout", + ); + assertStringIncludes( + (error as Error).message, + "running", + "Terminal run polling deadline should include the last observed state", + ); + }, + ); + }); + + it("aborts each poll fetch independently", async () => { + const observed: AbortSignal[] = []; + await withMockFetch( + ((_input: URL | RequestInfo, init?: RequestInit) => { + assert( + init?.signal instanceof AbortSignal, + "Terminal run polling should attach an AbortSignal to each fetch", + ); + observed.push(init.signal); + return Promise.resolve(Response.json({ status: "failed" })); + }) as typeof fetch, + async () => { + await waitForTerminalRun(new URL("http://127.0.0.1/runs/1"), 1_000); + assertEquals( + observed.length, + 1, + "Terminal run polling should issue exactly one fetch for immediate failure", + ); + assertEquals( + observed[0]?.aborted, + false, + "Terminal run polling should clear the timeout without aborting a completed fetch", + ); + }, + ); + }); + + it("requires the listed run to carry the failed provider node state", () => { + const listed = assertListedRunFailure( + "node/packed npm consumer", + { + runs: [{ + id: "run-1", + status: "failed", + nodeStates: { + "call-provider": { + status: "failed", + error: "Agent timed out after 2000ms", + }, + }, + }], + }, + "run-1", + ); + + assertEquals( + listed.id, + "run-1", + "List assertion should return the same run that matched the requested run id", + ); + + assertThrows( + () => + assertListedRunFailure( + "node/packed npm consumer", + { runs: [] }, + "run-1", + ), + Error, + "persistence/list: failed run was not listed", + "List assertion should reject a response that omits the requested run", + ); + assertThrows( + () => + assertListedRunFailure( + "node/packed npm consumer", + { + runs: [{ + id: "run-1", + status: "running", + nodeStates: { "call-provider": { status: "running" } }, + }], + }, + "run-1", + ), + Error, + "persistence/list: listed run was not failed", + "List assertion should reject a requested run that is not failed", + ); + + assertThrows( + () => + assertListedRunFailure( + "node/packed npm consumer", + { + runs: [{ + id: "run-1", + status: "failed", + nodeStates: { "call-provider": { status: "running" } }, + }], + }, + "run-1", + ), + Error, + "persistence/list: call-provider was not failed", + "List assertion should reject a listed run whose provider node is not failed", + ); + assertThrows( + () => + assertListedRunFailure( + "node/packed npm consumer", + { + runs: [{ + id: "run-1", + status: "failed", + nodeStates: { "call-provider": { status: "failed" } }, + }], + }, + "run-1", + ), + Error, + "expected 2000ms timeout evidence for call-provider", + "List assertion should reject a failed provider node with missing timeout error evidence", + ); + assertThrows( + () => + assertListedRunFailure( + "node/packed npm consumer", + { + runs: [{ + id: "run-1", + status: "failed", + nodeStates: { + "call-provider": { + status: "failed", + error: "provider returned 401", + }, + }, + }], + }, + "run-1", + ), + Error, + "expected 2000ms timeout evidence for call-provider", + "List assertion should reject a failed provider node with non-timeout error evidence", + ); + }); + + it("classifies successful plaintext start responses as route/start JSON failures", () => { + assertThrows( + () => + parseScopedResponseJson( + "node/packed npm consumer", + "route/start", + "this is not json", + ), + Error, + "node/packed npm consumer route/start: response was not JSON: this is not json", + "Route/start JSON parser should preserve the route/start failure scope", + ); + }); + + it("classifies successful plaintext list responses as persistence/list JSON failures", () => { + assertThrows( + () => + parseScopedResponseJson( + "bun/packed package consumer", + "persistence/list", + "plain text list response", + ), + Error, + "bun/packed package consumer persistence/list: response was not JSON: plain text list response", + "Persistence/list JSON parser should preserve the list failure scope", + ); + }); + + it("does not execute the critical-flow journey on import", async () => { + const exports = await inspectModuleExports( + new URL("./runtime-inference-critical-flow.ts", import.meta.url), + "runtime inference", + ); + assert( + exports.includes("parseRuntimeSelection"), + "Critical-flow module should export parseRuntimeSelection on import", + ); + assert( + exports.includes("runRuntimeInferenceCriticalFlow"), + "Critical-flow module should export runRuntimeInferenceCriticalFlow on import", + ); + }); +}); + +describe("runtime inference critical-flow CI contract", () => { + it("exposes the focused script test and executable runtime task", async () => { + const denoConfig = JSON.parse( + await Deno.readTextFile(new URL("deno.json", REPO_ROOT)), + ) as { + tasks: Record; + }; + + const testScripts = String(denoConfig.tasks["test:scripts"]); + assertStringIncludes( + testScripts, + FLOW_TEST_PATH, + "test:scripts should include the critical-flow unit test", + ); + + const task = denoConfig.tasks["test:e2e:runtime-inference-critical-flow"]; + assertEquals( + typeof task, + "string", + "Critical-flow E2E task should be registered as a string command", + ); + assertStringIncludes( + String(task), + `deno run --allow-all ${FLOW_HARNESS_PATH}`, + "Critical-flow E2E task should execute the shared TypeScript harness", + ); + }); + + it("runs a dedicated runtime critical-flow matrix with stable check names", async () => { + const workflow = yamlRecord( + parse( + await Deno.readTextFile( + new URL(".github/workflows/cicd.yml", REPO_ROOT), + ), + ), + "cicd workflow", + ); + const jobs = yamlRecord(workflow.jobs, "cicd workflow jobs"); + const job = yamlRecord( + jobs["tests-runtime-critical-flow"], + "tests-runtime-critical-flow job", + ); + + assertEquals( + job.name, + "tests (runtime critical flow: ${{ matrix.runtime }})", + "Runtime critical-flow job should expose stable matrix check names", + ); + assertEquals( + job["runs-on"], + "ubuntu-latest", + "Runtime critical-flow job should run on Ubuntu", + ); + assertEquals( + job["timeout-minutes"], + 20, + "Runtime critical-flow job should keep a bounded timeout", + ); + + const strategy = yamlRecord(job.strategy, "runtime critical-flow strategy"); + const matrix = yamlRecord(strategy.matrix, "runtime critical-flow matrix"); + assertEquals( + matrix.runtime, + ["deno", "node", "bun"], + "Runtime critical-flow matrix should include Deno, Node, and Bun lanes", + ); + + const steps = job.steps as Array>; + assert( + steps.some((step) => step.uses === "./.github/actions/setup-deno"), + "Runtime critical-flow job should install Deno", + ); + assert( + steps.some((step) => + step.uses === + "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020" && + yamlRecord(step.with, "setup-node with")["node-version"] === "24" + ), + "Runtime critical-flow job should install Node 24", + ); + assert( + steps.some((step) => + step.uses === + "oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6" && + yamlRecord(step.with, "setup-bun with")["bun-version"] === "1.3.6" + ), + "Runtime critical-flow job should install Bun 1.3.6 for the Bun lane", + ); + assert( + steps.some((step) => + step.name === "Run runtime critical flow" && + step.run === + "deno task test:e2e:runtime-inference-critical-flow --runtime=${{ matrix.runtime }}" + ), + "Runtime critical-flow job should invoke the shared task with the matrix runtime", + ); + }); +}); diff --git a/scripts/test/runtime-inference-critical-flow.ts b/scripts/test/runtime-inference-critical-flow.ts new file mode 100644 index 0000000000..f58c29cf4c --- /dev/null +++ b/scripts/test/runtime-inference-critical-flow.ts @@ -0,0 +1,980 @@ +import { + allocatePort, + assertCondition, + ensureCommand, + installDependencies, + packNpmPackage, + parseCommaSeparatedFlag, + runChecked, + type RuntimeName, + scaffoldProject, + startDevServer, + stopDevServer, + waitForRoute, +} from "./runtime-e2e-helpers.ts"; + +export type { RuntimeName } from "./runtime-e2e-helpers.ts"; + +export interface WorkflowRunDetail { + id?: string; + status?: string; + input?: Record; + nodeStates?: Record; + error?: unknown; +} + +type ProviderMode = "black-hole" | "respond"; + +const VALID_RUNTIMES: RuntimeName[] = ["node", "bun", "deno"]; +const AGENT_MODEL = "anthropic/claude-haiku-4-5-20251001"; +const ANTHROPIC_WIRE_MODEL = "claude-haiku-4-5-20251001"; +const ANTHROPIC_API_KEY = "vf-runtime-critical-flow-key"; +const WORKFLOW_ID = "content-pipeline"; +const NODE_ID = "call-provider"; +const APPLICATION_ROUTE_PAYLOAD = { + ok: true, + surface: "runtime-critical-flow", +}; +const POLL_REQUEST_TIMEOUT_MS = 1_000; +const TERMINAL_STATUSES = new Set(["failed", "completed", "cancelled"]); + +class UnexpectedTerminalRunError extends Error {} +class RunTerminatedBeforeProviderReceiptError extends Error {} + +export function parseRuntimeSelection(args: string[]): RuntimeName[] { + const requested = parseCommaSeparatedFlag(args, ["runtime", "runtimes"]); + if (!requested) { + return [...VALID_RUNTIMES]; + } + + const seen = new Set(); + const runtimes: RuntimeName[] = []; + for (const runtime of requested) { + if (!VALID_RUNTIMES.includes(runtime as RuntimeName)) { + throw new Error(`Unknown runtime: ${runtime}`); + } + if (seen.has(runtime)) { + throw new Error(`Duplicate runtime: ${runtime}`); + } + seen.add(runtime); + runtimes.push(runtime as RuntimeName); + } + + return runtimes; +} + +export function artifactClaim(runtime: RuntimeName): string { + switch (runtime) { + case "node": + return "packed npm consumer"; + case "bun": + return "packed package consumer"; + case "deno": + return "packed CLI"; + } +} + +function bodyContainsMarker(value: unknown, marker: string): boolean { + if (typeof value === "string") { + return value.includes(marker); + } + if (Array.isArray(value)) { + return value.some((entry) => bodyContainsMarker(entry, marker)); + } + if (value && typeof value === "object") { + return Object.values(value).some((entry) => + bodyContainsMarker(entry, marker) + ); + } + return false; +} + +export async function validateAnthropicRequest( + request: Request, + expectedMarker: string, +): Promise { + await validateAnthropicRequestMarkers(request, [expectedMarker]); +} + +async function validateAnthropicRequestMarkers( + request: Request, + expectedMarkers: string[], +): Promise<{ marker: string; stream: boolean }> { + const url = new URL(request.url); + if (request.method !== "POST" || url.pathname !== "/v1/messages") { + throw new Error( + `Expected Anthropic request to be POST /v1/messages, got ${request.method} ${url.pathname}`, + ); + } + + if (request.headers.get("x-api-key") !== ANTHROPIC_API_KEY) { + throw new Error( + "Anthropic request did not include the expected x-api-key header", + ); + } + + if (request.headers.get("anthropic-version") !== "2023-06-01") { + throw new Error( + "Anthropic request did not include anthropic-version 2023-06-01", + ); + } + + const contentType = request.headers.get("content-type")?.split(";", 1)[0] + ?.trim().toLowerCase(); + if (contentType !== "application/json") { + throw new Error( + `Anthropic request content-type must be application/json, got ${ + contentType ?? "missing" + }`, + ); + } + + let body: unknown; + try { + body = await request.json(); + } catch { + throw new Error("Anthropic request body was not valid JSON"); + } + + const model = body && typeof body === "object" + ? (body as { model?: unknown }).model + : undefined; + if (model !== ANTHROPIC_WIRE_MODEL) { + throw new Error( + `Expected Anthropic model ${ANTHROPIC_WIRE_MODEL}, got ${String(model)}`, + ); + } + + const matchedMarker = expectedMarkers.find((marker) => + bodyContainsMarker(body, marker) + ); + if (matchedMarker === undefined) { + throw new Error( + `Anthropic request body did not include marker ${ + expectedMarkers.join(" or ") + }`, + ); + } + + const stream = body && typeof body === "object" + ? (body as { stream?: unknown }).stream === true + : false; + return { marker: matchedMarker, stream }; +} + +function isProviderValidationFailure( + error: Error, + state: ProviderState, +): boolean { + return state.validationFailure() === error; +} + +async function fetchWithTimeout(url: URL): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), POLL_REQUEST_TIMEOUT_MS); + try { + return await fetch(url, { signal: controller.signal }); + } finally { + clearTimeout(timeout); + } +} + +export async function waitForTerminalRun( + url: URL, + deadlineMs: number, +): Promise { + const deadline = Date.now() + deadlineMs; + let lastObservation = "none"; + + while (Date.now() < deadline) { + try { + const response = await fetchWithTimeout(url); + const body = await response.text(); + if (!response.ok) { + lastObservation = `HTTP ${response.status}: ${body.slice(0, 500)}`; + } else { + const detail = JSON.parse(body) as WorkflowRunDetail; + lastObservation = JSON.stringify(detail).slice(0, 1_000); + if (detail.status && TERMINAL_STATUSES.has(detail.status)) { + if (detail.status !== "failed") { + throw new UnexpectedTerminalRunError( + `Run reached unexpected terminal status ${detail.status}: ${lastObservation}`, + ); + } + return detail; + } + } + } catch (error) { + if ( + error instanceof UnexpectedTerminalRunError + ) { + throw error; + } + lastObservation = error instanceof Error ? error.message : String(error); + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + throw new Error( + `Run timed out before reaching terminal failure within ${deadlineMs}ms deadline; last observation: ${lastObservation}`, + ); +} + +export function parseScopedResponseJson( + label: string, + scope: "route/start" | "route/application-api" | "persistence/list", + body: string, +): T { + try { + return JSON.parse(body) as T; + } catch { + throw new Error( + `${label} ${scope}: response was not JSON: ${body.slice(0, 500)}`, + ); + } +} + +function hasFlag(args: string[], name: string): boolean { + return args.includes(`--${name}`); +} + +function parseProviderMode(args: string[]): ProviderMode { + const value = parseCommaSeparatedFlag(args, ["provider-mode"])?.[0] ?? + "black-hole"; + if (value !== "black-hole" && value !== "respond") { + throw new Error(`Unknown provider mode: ${value}`); + } + return value; +} + +async function writeFixture(projectDir: string): Promise { + await Deno.remove(`${projectDir}/app/api/workflows`, { recursive: true }) + .catch( + () => {}, + ); + await Deno.mkdir(`${projectDir}/app/api/workflows/[...path]`, { + recursive: true, + }); + await Deno.mkdir(`${projectDir}/app/api/ag-ui`, { recursive: true }); + await Deno.mkdir(`${projectDir}/app/api/critical-path`, { recursive: true }); + await Deno.mkdir(`${projectDir}/agents`, { recursive: true }); + await Deno.mkdir(`${projectDir}/lib`, { recursive: true }); + await Deno.mkdir(`${projectDir}/workflows`, { recursive: true }); + + await Deno.writeTextFile( + `${projectDir}/agents/content-agent.ts`, + `import { agent } from "veryfront/agent"; + +export default agent({ + id: "content-agent", + model: "${AGENT_MODEL}", + system: "Return the marker you receive.", + maxSteps: 1, +}); +`, + ); + await Deno.writeTextFile( + `${projectDir}/workflows/content-pipeline.ts`, + `import { agentStep, workflow } from "veryfront/workflow"; +import { defineSchema } from "veryfront/schemas"; + +export default workflow({ + id: "${WORKFLOW_ID}", + inputSchema: defineSchema((v) => v.object({ marker: v.string() }))(), + steps: [ + agentStep("${NODE_ID}", "content-agent", { + input: (context) => \`marker: \${(context.input as { marker: string }).marker}\`, + timeout: "2s", + }), + ], +}); +`, + ); + await Deno.writeTextFile( + `${projectDir}/lib/workflows.ts`, + `import { getAgent, getAllAgentIds } from "veryfront/agent"; +import { toolRegistry } from "veryfront/tool"; +import { createWorkflowClient, MemoryBackend, type Workflow } from "veryfront/workflow"; +import "../agents/content-agent.ts"; +import contentPipeline from "../workflows/content-pipeline.ts"; + +const globalScope = globalThis as typeof globalThis & { + runtimeCriticalFlowWorkflowClient?: ReturnType; +}; + +export const workflows = globalScope.runtimeCriticalFlowWorkflowClient ??= createWorkflowClient({ + backend: new MemoryBackend(), + executor: { + stepExecutor: { + agentRegistry: { get: getAgent, list: getAllAgentIds }, + toolRegistry, + }, + }, +}); + +workflows.register(contentPipeline as Workflow); +`, + ); + await Deno.writeTextFile( + `${projectDir}/app/api/ag-ui/route.ts`, + `import { createAgUiHandler } from "veryfront/agent"; +import "../../../agents/content-agent.ts"; + +export const POST = createAgUiHandler("content-agent"); +`, + ); + await Deno.writeTextFile( + `${projectDir}/app/api/critical-path/route.ts`, + `export function GET(): Response { + return Response.json(${JSON.stringify(APPLICATION_ROUTE_PAYLOAD)}); +} +`, + ); + await Deno.writeTextFile( + `${projectDir}/app/api/workflows/[...path]/route.ts`, + `import { createWorkflowHandler } from "veryfront/workflow"; +import { workflows } from "../../../../lib/workflows.ts"; + +export const { GET, POST } = createWorkflowHandler(workflows); +`, + ); +} + +interface ProviderState { + received: string[]; + server: Deno.HttpServer; + abort(): void; + closed: Promise; + validationFailure(): Error | undefined; + cancellationEvidence(): string | undefined; + cancellation: Promise; + url: URL; +} + +function stringifyError(value: unknown): string { + return typeof value === "string" ? value : String(JSON.stringify(value)); +} + +export function parseAgUiTextDeltas(body: string): string[] { + const deltas: string[] = []; + for (const frame of body.split(/\r?\n\r?\n/)) { + const lines = frame.split(/\r?\n/); + const event = lines.find((line) => line.startsWith("event:")) + ?.slice("event:".length).trim(); + if (event !== "TextMessageContent") continue; + + const data = lines.filter((line) => line.startsWith("data:")) + .map((line) => line.slice("data:".length).trimStart()).join("\n"); + let payload: unknown; + try { + payload = JSON.parse(data); + } catch { + throw new Error("AG-UI TextMessageContent event contained invalid JSON"); + } + const delta = payload && typeof payload === "object" + ? (payload as { delta?: unknown }).delta + : undefined; + if (typeof delta !== "string") { + throw new Error("AG-UI TextMessageContent event omitted a string delta"); + } + deltas.push(delta); + } + return deltas; +} + +function anthropicStreamResponse(marker: string): Response { + const messageId = `msg_${crypto.randomUUID().replaceAll("-", "")}`; + const events = [ + [ + "message_start", + { + type: "message_start", + message: { + id: messageId, + type: "message", + role: "assistant", + model: ANTHROPIC_WIRE_MODEL, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }, + ], + [ + "content_block_start", + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }, + ], + [ + "content_block_delta", + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: marker }, + }, + ], + ["content_block_stop", { type: "content_block_stop", index: 0 }], + [ + "message_delta", + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + ], + ["message_stop", { type: "message_stop" }], + ] as const; + const body = events.map(([event, data]) => + `event: ${event}\ndata: ${JSON.stringify(data)}\n\n` + ) + .join(""); + + return new Response(body, { + headers: { "content-type": "text/event-stream; charset=utf-8" }, + }); +} + +function anthropicJsonResponse(marker: string): Response { + return Response.json({ + id: `msg_${crypto.randomUUID().replaceAll("-", "")}`, + type: "message", + role: "assistant", + model: ANTHROPIC_WIRE_MODEL, + content: [{ type: "text", text: marker }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); +} + +export function assertListedRunFailure( + label: string, + list: { runs?: WorkflowRunDetail[] }, + runId: string, +): WorkflowRunDetail { + const listed = list.runs?.find((run) => run.id === runId); + if (listed === undefined) { + throw new Error( + `${label} persistence/list: failed run was not listed: ${ + JSON.stringify(list) + }`, + ); + } + if (listed.status !== "failed") { + throw new Error( + `${label} persistence/list: listed run was not failed: ${ + JSON.stringify(listed) + }`, + ); + } + + const node = listed.nodeStates?.[NODE_ID]; + if (node?.status !== "failed") { + throw new Error( + `${label} persistence/list: ${NODE_ID} was not failed: ${ + JSON.stringify(listed) + }`, + ); + } + + const nodeError = stringifyError(node.error); + assertCondition( + nodeError.includes("timed out after 2000ms"), + `${label} persistence/list: expected 2000ms timeout evidence for ${NODE_ID}, got ${nodeError}`, + ); + + return listed; +} + +export async function waitForProviderCancellation( + state: ProviderState, + runtime: RuntimeName, + timeoutMs = 1_500, +): Promise { + if (state.cancellationEvidence()) return; + + let timeoutId: number | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout( + () => + reject( + new Error( + `${runtime}/${ + artifactClaim(runtime) + } provider/cancellation: provider request was not aborted by the client before cleanup within ${timeoutMs}ms`, + ), + ), + timeoutMs, + ); + }); + + try { + await Promise.race([state.cancellation, timeoutPromise]); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } +} + +function startProvider( + agentMarker: string, + workflowMarker: string, + mode: ProviderMode, +): ProviderState { + const received: string[] = []; + const controller = new AbortController(); + let resolveClosed!: () => void; + const closed = new Promise((resolve) => { + resolveClosed = resolve; + }); + let validationFailure: Error | undefined; + let cancellationEvidence: string | undefined; + let resolveCancellation!: (evidence: string) => void; + const cancellation = new Promise((resolve) => { + resolveCancellation = resolve; + }); + const pendingResponses = new Set<() => void>(); + const recordCancellation = (evidence: string) => { + cancellationEvidence ??= evidence; + resolveCancellation(cancellationEvidence); + }; + const server = Deno.serve({ + hostname: "127.0.0.1", + port: 0, + signal: controller.signal, + onListen: () => {}, + }, async (request) => { + let validated: { marker: string; stream: boolean }; + try { + validated = await validateAnthropicRequestMarkers(request.clone(), [ + agentMarker, + workflowMarker, + ]); + } catch (error) { + validationFailure = error instanceof Error + ? error + : new Error(String(error)); + return Response.json({ error: validationFailure.message }, { + status: 400, + }); + } + const { marker, stream } = validated; + received.push(marker); + + if (marker === agentMarker || mode === "respond") { + return stream + ? anthropicStreamResponse(marker) + : anthropicJsonResponse(marker); + } + + return await new Promise((resolve) => { + const release = () => resolve(new Response(null, { status: 499 })); + const onClientAbort = () => { + recordCancellation("client connection aborted provider request"); + }; + pendingResponses.add(release); + controller.signal.addEventListener("abort", release, { once: true }); + if (request.signal.aborted) { + onClientAbort(); + } else { + request.signal.addEventListener("abort", onClientAbort, { + once: true, + }); + } + }); + }); + + server.finished.finally(resolveClosed); + + return { + received, + server, + abort() { + controller.abort(); + for (const release of pendingResponses) release(); + pendingResponses.clear(); + }, + closed, + validationFailure() { + return validationFailure; + }, + cancellationEvidence() { + return cancellationEvidence; + }, + cancellation, + url: new URL(`http://127.0.0.1:${server.addr.port}/v1`), + }; +} + +export async function waitForProviderReceipt( + state: ProviderState, + runtime: RuntimeName, + detailUrl: URL, + expectedMarker?: string, +): Promise { + const deadline = Date.now() + 5_000; + let lastRun = "none"; + while (Date.now() < deadline) { + const validationFailure = state.validationFailure(); + if (validationFailure) throw validationFailure; + if ( + expectedMarker === undefined + ? state.received.length > 0 + : state.received.includes(expectedMarker) + ) return; + try { + const response = await fetch(detailUrl, { + signal: AbortSignal.timeout(1_000), + }); + const body = await response.text(); + lastRun = `HTTP ${response.status}: ${body.slice(0, 1_000)}`; + if (response.ok) { + const detail = JSON.parse(body) as WorkflowRunDetail; + const validationFailure = state.validationFailure(); + if (validationFailure) throw validationFailure; + if (detail.status && TERMINAL_STATUSES.has(detail.status)) { + if ( + expectedMarker === undefined + ? state.received.length > 0 + : state.received.includes(expectedMarker) + ) return; + throw new RunTerminatedBeforeProviderReceiptError( + `${runtime}/${ + artifactClaim(runtime) + } provider/request: run terminated before provider receipt: ${ + JSON.stringify(detail) + }`, + ); + } + } + } catch (error) { + if ( + error instanceof Error && + isProviderValidationFailure(error, state) + ) { + throw error; + } + if ( + error instanceof RunTerminatedBeforeProviderReceiptError + ) { + throw error; + } + lastRun = error instanceof Error ? error.message : String(error); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error( + `${runtime}/${ + artifactClaim(runtime) + } provider/request: provider was not reached; last run: ${lastRun}`, + ); +} + +function scopedLogs(server: { stdout: string[]; stderr: string[] }): string { + const stdout = server.stdout.join("").slice(-4_000).trim(); + const stderr = server.stderr.join("").slice(-4_000).trim(); + return [ + stdout ? `stdout:\n${stdout}` : "", + stderr ? `stderr:\n${stderr}` : "", + ] + .filter(Boolean) + .join("\n\n"); +} + +async function postJson(url: URL, body: unknown): Promise { + return await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000), + }); +} + +async function assertApplicationPage( + rootUrl: URL, + label: string, + scope: "route/page" | "server/post-timeout", +): Promise { + const response = await fetch(rootUrl, { signal: AbortSignal.timeout(5_000) }); + const body = await response.text(); + assertCondition( + response.ok, + `${label} ${scope}: root returned HTTP ${response.status}`, + ); + assertCondition( + body.includes("Content Pipeline"), + `${label} ${scope}: root omitted expected application content`, + ); +} + +async function assertApplicationApi( + rootUrl: URL, + label: string, +): Promise { + const response = await fetch(new URL("/api/critical-path", rootUrl), { + signal: AbortSignal.timeout(5_000), + }); + const body = await response.text(); + assertCondition( + response.ok, + `${label} route/application-api: HTTP ${response.status} ${ + body.slice(0, 500) + }`, + ); + const parsed = parseScopedResponseJson>( + label, + "route/application-api", + body, + ); + assertCondition( + JSON.stringify(parsed) === JSON.stringify(APPLICATION_ROUTE_PAYLOAD), + `${label} route/application-api: unexpected payload ${body.slice(0, 500)}`, + ); +} + +async function assertAgentRoute( + rootUrl: URL, + label: string, + marker: string, +): Promise { + const response = await postJson(new URL("/api/ag-ui", rootUrl), { + messages: [{ + id: `message-${crypto.randomUUID()}`, + role: "user", + parts: [{ type: "text", text: marker }], + }], + }); + const body = await response.text(); + assertCondition( + response.ok, + `${label} route/agent: HTTP ${response.status} ${body.slice(0, 500)}`, + ); + assertCondition( + response.headers.get("content-type")?.includes("text/event-stream") === + true, + `${label} route/agent: expected text/event-stream response`, + ); + assertCondition( + parseAgUiTextDeltas(body).join("").includes(marker), + `${label} route/agent: assistant text omitted provider marker`, + ); + assertCondition( + body.includes("event: RunFinished"), + `${label} route/agent: response omitted terminal RunFinished event`, + ); +} + +async function assertRuntimeJourney( + workDir: string, + tarballPath: string, + runtime: RuntimeName, + providerMode: ProviderMode, +): Promise { + const label = `${runtime}/${artifactClaim(runtime)}`; + const scenarioId = `${runtime}-${crypto.randomUUID()}`; + const agentMarker = `runtime-critical-agent-${scenarioId}`; + const workflowMarker = `runtime-critical-workflow-${scenarioId}`; + const provider = startProvider(agentMarker, workflowMarker, providerMode); + let server: + | ReturnType + | undefined; + try { + console.log(`${label}: scaffold`); + const projectDir = await scaffoldProject( + workDir, + tarballPath, + "agentic-workflow", + runtime, + ); + await writeFixture(projectDir); + + console.log(`${label}: install`); + await installDependencies(projectDir, runtime, workDir); + + const port = allocatePort(); + server = startDevServer(projectDir, runtime, port, { + ANTHROPIC_API_KEY, + ANTHROPIC_BASE_URL: provider.url.toString(), + VERYFRONT_HOST_ALLOW_INTERNAL_EGRESS: "true", + }); + + const rootUrl = new URL(`http://127.0.0.1:${port}/`); + console.log(`${label}: readiness ${rootUrl}`); + await waitForRoute(rootUrl.toString(), 60_000); + await assertApplicationPage(rootUrl, label, "route/page"); + await assertApplicationApi(rootUrl, label); + await assertAgentRoute(rootUrl, label, agentMarker); + assertCondition( + provider.received.filter((marker) => marker === agentMarker).length === 1, + `${label} provider/request: expected exactly one direct-agent request`, + ); + + const startUrl = new URL(`/api/workflows/${WORKFLOW_ID}/start`, rootUrl); + const startedAt = Date.now(); + const startResponse = await postJson(startUrl, { + input: { marker: workflowMarker }, + }); + const startBody = await startResponse.text(); + assertCondition( + startResponse.ok, + `${label} route/start: HTTP ${startResponse.status} ${ + startBody.slice(0, 500) + }`, + ); + const started = parseScopedResponseJson<{ + runId?: unknown; + id?: unknown; + }>(label, "route/start", startBody); + const runId = started.runId ?? started.id; + if (typeof runId !== "string" || runId.length === 0) { + throw new Error(`${label} route/start: response omitted runId`); + } + const runIdString = runId; + + const detailUrl = new URL(`/api/workflows/runs/${runIdString}`, rootUrl); + await waitForProviderReceipt( + provider, + runtime, + detailUrl, + workflowMarker, + ); + assertCondition( + provider.received.filter((marker) => marker === workflowMarker).length === + 1, + `${label} provider/request: expected exactly one workflow request`, + ); + + const detail = await waitForTerminalRun( + detailUrl, + 10_000, + ); + const elapsedMs = Date.now() - startedAt; + + if (providerMode === "respond") { + throw new Error( + `${label} negative control expected timeout failure, but provider-mode=respond should complete: ${ + JSON.stringify(detail) + }`, + ); + } + + assertCondition( + detail.status === "failed", + `${label} persistence/detail: run was not failed`, + ); + const node = detail.nodeStates?.[NODE_ID]; + if (node?.status !== "failed") { + throw new Error( + `${label} persistence/detail: ${NODE_ID} was not failed: ${ + JSON.stringify(detail) + }`, + ); + } + const nodeError = stringifyError(node.error); + assertCondition( + nodeError.includes("timed out after 2000ms"), + `${label} timeout/cancellation: expected 2000ms timeout evidence, got ${nodeError}`, + ); + assertCondition( + elapsedMs >= 1_750 && elapsedMs < 10_000, + `${label} timeout/cancellation: elapsed ${elapsedMs}ms outside expected bounds`, + ); + await waitForProviderCancellation(provider, runtime); + + const listResponse = await fetch( + new URL(`/api/workflows/runs?workflowId=${WORKFLOW_ID}`, rootUrl), + { signal: AbortSignal.timeout(5_000) }, + ); + const listBody = await listResponse.text(); + assertCondition( + listResponse.ok, + `${label} persistence/list: HTTP ${listResponse.status} ${ + listBody.slice(0, 500) + }`, + ); + const list = parseScopedResponseJson<{ runs?: WorkflowRunDetail[] }>( + label, + "persistence/list", + listBody, + ); + assertListedRunFailure(label, list, runIdString); + + await assertApplicationPage(rootUrl, label, "server/post-timeout"); + } catch (error) { + if (server) await stopDevServer(server); + const logs = server ? scopedLogs(server) : ""; + throw new Error( + [ + `${label} failed`, + error instanceof Error ? error.message : String(error), + logs, + ].filter(Boolean).join("\n\n"), + ); + } finally { + if (server) await stopDevServer(server); + provider.abort(); + await provider.server.shutdown().catch(() => {}); + await provider.closed.catch(() => {}); + } +} + +export async function runRuntimeInferenceCriticalFlow( + args = Deno.args, +): Promise { + const rootDir = new URL("../../", import.meta.url).pathname.replace( + /\/$/, + "", + ); + const runtimes = parseRuntimeSelection(args); + const providerMode = parseProviderMode(args); + const keepWorkDir = hasFlag(args, "keep"); + const skipBuild = hasFlag(args, "skip-build"); + const workDir = await Deno.makeTempDir({ + prefix: "veryfront-runtime-critical-flow-", + }); + + console.log(`runtimes: ${runtimes.join(", ")}`); + console.log(`provider mode: ${providerMode}`); + + try { + await ensureCommand("npm"); + await ensureCommand("node"); + if (runtimes.includes("bun")) await ensureCommand("bun"); + if (runtimes.includes("deno")) await ensureCommand("deno"); + + if (!skipBuild) { + console.log("build npm package"); + await runChecked("deno", ["task", "build:npm"], { + cwd: rootDir, + timeoutMs: 300_000, + }); + } + + console.log("pack npm package"); + const tarballPath = await packNpmPackage(rootDir, workDir); + + for (const runtime of runtimes) { + await assertRuntimeJourney( + workDir, + tarballPath, + runtime, + providerMode, + ); + console.log(`${runtime}/${artifactClaim(runtime)}: passed`); + } + } finally { + if (keepWorkDir) { + console.log(`kept work dir: ${workDir}`); + } else { + await Deno.remove(workDir, { recursive: true }).catch(() => {}); + } + } +} + +if (import.meta.main) { + await runRuntimeInferenceCriticalFlow(); +} diff --git a/scripts/test/template-runtime-e2e.test.ts b/scripts/test/template-runtime-e2e.test.ts index ee805da316..15591be99b 100644 --- a/scripts/test/template-runtime-e2e.test.ts +++ b/scripts/test/template-runtime-e2e.test.ts @@ -1,23 +1,147 @@ import { assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { getDevServerCommand } from "./template-runtime-e2e.ts"; +import { + getDevServerEnvironment, + inspectModuleExports, + startDevServer, + stopDevServer, +} from "./runtime-e2e-helpers.ts"; describe("template runtime E2E commands", () => { + it("exports the shared harness without running the E2E flow on import", async () => { + assertEquals( + await inspectModuleExports( + new URL("./template-runtime-e2e.ts", import.meta.url), + "template runtime", + ), + [ + "assertCondition", + "ensureCommand", + "getDevServerCommand", + "installDependencies", + "packNpmPackage", + "parseCommaSeparatedFlag", + "runChecked", + "scaffoldProject", + "startDevServer", + "stopDevServer", + "waitForRoute", + ], + "Template runtime module should export only shared harness helpers on import", + ); + }); + it("passes the selected port through Deno task without a separator", () => { - assertEquals(getDevServerCommand("deno", 4321), { - command: "deno", - args: ["task", "dev", "--port", "4321"], - }); + assertEquals( + getDevServerCommand("deno", 4321), + { + command: "deno", + args: ["task", "dev", "--port", "4321"], + }, + "Deno dev command should pass the selected port directly to the task", + ); }); it("preserves the script argument separator for npm and Bun", () => { - assertEquals(getDevServerCommand("node", 4321), { - command: "npm", - args: ["run", "dev", "--", "--port", "4321"], - }); - assertEquals(getDevServerCommand("bun", 4321), { - command: "bun", - args: ["run", "dev", "--", "--port", "4321"], + assertEquals( + getDevServerCommand("node", 4321), + { + command: "npm", + args: ["run", "dev", "--", "--port", "4321"], + }, + "Node dev command should preserve npm's script argument separator", + ); + assertEquals( + getDevServerCommand("bun", 4321), + { + command: "bun", + args: ["run", "dev", "--", "--port", "4321"], + }, + "Bun dev command should preserve Bun's script argument separator", + ); + }); + + it("merges child-only environment overrides with deterministic defaults", () => { + assertEquals( + getDevServerEnvironment({ + ANTHROPIC_API_KEY: "test-key", + ANTHROPIC_BASE_URL: "http://127.0.0.1:4312/v1", + }), + { + ANTHROPIC_API_KEY: "test-key", + GOOGLE_API_KEY: "", + GOOGLE_GENERATIVE_AI_API_KEY: "", + LOG_FORMAT: "text", + MISTRAL_API_KEY: "", + NODE_ENV: "development", + OPENAI_API_KEY: "", + REVALIDATION_PER_PROJECT_LIMIT: "0", + SSR_TRANSFORM_PER_PROJECT_LIMIT: "0", + VF_DISABLE_LRU_INTERVAL: "1", + VERYFRONT_API_TOKEN: "", + ANTHROPIC_BASE_URL: "http://127.0.0.1:4312/v1", + }, + "Dev server environment should blank competing credentials and isolate scenario overrides in the child process", + ); + }); + + it("terminates npm script descendants when stopping a Node dev server", async () => { + if (Deno.build.os === "windows") return; + + const projectDir = await Deno.makeTempDir({ + prefix: "veryfront-runtime-server-cleanup-", }); + const pidFile = `${projectDir}/server.pid`; + let descendantPid: number | undefined; + let server: ReturnType | undefined; + + try { + await Deno.writeTextFile( + `${projectDir}/package.json`, + `${JSON.stringify({ scripts: { dev: "node server.mjs" } })}\n`, + ); + await Deno.writeTextFile( + `${projectDir}/server.mjs`, + [ + 'import { writeFileSync } from "node:fs";', + "writeFileSync(process.env.RUNTIME_E2E_PID_FILE, String(process.pid));", + 'process.on("SIGTERM", () => {});', + "setTimeout(() => {}, 10_000);", + ].join("\n"), + ); + server = startDevServer(projectDir, "node", 4321, { + RUNTIME_E2E_PID_FILE: pidFile, + }); + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + try { + descendantPid = Number(await Deno.readTextFile(pidFile)); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + } + assertEquals(Number.isSafeInteger(descendantPid), true); + + await stopDevServer(server); + assertEquals(await isProcessAlive(descendantPid!), false); + } finally { + if (server) await stopDevServer(server).catch(() => {}); + if (descendantPid && await isProcessAlive(descendantPid)) { + Deno.kill(descendantPid, "SIGKILL"); + } + await Deno.remove(projectDir, { recursive: true }).catch(() => {}); + } }); }); + +async function isProcessAlive(pid: number): Promise { + const status = await new Deno.Command("kill", { + args: ["-0", String(pid)], + stdout: "null", + stderr: "null", + }).output(); + return status.success; +} diff --git a/scripts/test/template-runtime-e2e.ts b/scripts/test/template-runtime-e2e.ts index 118ee30b50..5c6c645385 100644 --- a/scripts/test/template-runtime-e2e.ts +++ b/scripts/test/template-runtime-e2e.ts @@ -1,13 +1,33 @@ import { TEMPLATES } from "../../cli/commands/init/catalog.ts"; - -type RuntimeName = "node" | "bun" | "deno"; -type TemplateName = typeof TEMPLATES[number]["id"]; - -interface CommandResult { - code: number; - stdout: string; - stderr: string; -} +import { + allocatePort, + assertCondition, + ensureCommand, + installDependencies, + packNpmPackage, + parseCommaSeparatedFlag, + runChecked, + type RuntimeName, + scaffoldProject, + startDevServer, + stopDevServer, + waitForRoute, +} from "./runtime-e2e-helpers.ts"; + +export { + assertCondition, + ensureCommand, + getDevServerCommand, + installDependencies, + packNpmPackage, + parseCommaSeparatedFlag, + runChecked, + scaffoldProject, + startDevServer, + stopDevServer, + waitForRoute, +} from "./runtime-e2e-helpers.ts"; +export type { CommandResult, RuntimeName } from "./runtime-e2e-helpers.ts"; interface BrowserEnvelope { success: boolean; @@ -30,6 +50,8 @@ interface BrowserRequest { error?: string | null; } +type TemplateName = typeof TEMPLATES[number]["id"]; + const VALID_RUNTIMES: RuntimeName[] = ["node", "bun", "deno"]; const DEFAULT_RUNTIMES: RuntimeName[] = VALID_RUNTIMES; const TEMPLATE_ROUTE_EXPECTATIONS: Partial< @@ -41,34 +63,12 @@ const TEMPLATE_ROUTE_EXPECTATIONS: Partial< ], }; -const decoder = new TextDecoder(); - -function parseCsvFlag(name: string): string[] | null { - const prefix = `--${name}=`; - const inline = Deno.args.find((arg) => arg.startsWith(prefix)); - if (inline) { - return inline.slice(prefix.length).split(",").map((value) => value.trim()) - .filter(Boolean); - } - - const index = Deno.args.indexOf(`--${name}`); - if (index >= 0) { - const value = Deno.args[index + 1]; - if (!value || value.startsWith("--")) { - throw new Error(`--${name} requires a comma-separated value`); - } - return value.split(",").map((entry) => entry.trim()).filter(Boolean); - } - - return null; -} - function hasFlag(name: string): boolean { return Deno.args.includes(`--${name}`); } function selectedTemplates(): TemplateName[] { - const requested = parseCsvFlag("templates"); + const requested = parseCommaSeparatedFlag(Deno.args, ["templates"]); const all = TEMPLATES.map((template) => template.id); if (!requested) { return [...all]; @@ -85,7 +85,7 @@ function selectedTemplates(): TemplateName[] { } function selectedRuntimes(): RuntimeName[] { - const requested = parseCsvFlag("runtimes"); + const requested = parseCommaSeparatedFlag(Deno.args, ["runtimes"]); if (!requested) { return DEFAULT_RUNTIMES; } @@ -100,187 +100,6 @@ function selectedRuntimes(): RuntimeName[] { return requested as RuntimeName[]; } -async function runCommand( - command: string, - args: string[], - options: { - cwd?: string; - env?: Record; - timeoutMs?: number; - } = {}, -): Promise { - const controller = new AbortController(); - const timeout = options.timeoutMs === undefined - ? undefined - : setTimeout(() => controller.abort(), options.timeoutMs); - - try { - const output = await new Deno.Command(command, { - args, - cwd: options.cwd, - env: options.env, - signal: controller.signal, - stdout: "piped", - stderr: "piped", - }).output(); - - return { - code: output.code, - stdout: decoder.decode(output.stdout), - stderr: decoder.decode(output.stderr), - }; - } catch (error) { - if (controller.signal.aborted) { - throw new Error( - `${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`, - ); - } - throw error; - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); - } - } -} - -async function runChecked( - command: string, - args: string[], - options: { - cwd?: string; - env?: Record; - timeoutMs?: number; - } = {}, -): Promise { - const result = await runCommand(command, args, options); - if (result.code !== 0) { - throw new Error( - [ - `${command} ${args.join(" ")} failed with exit code ${result.code}`, - result.stdout.trim(), - result.stderr.trim(), - ].filter(Boolean).join("\n"), - ); - } - return result; -} - -async function ensureCommand( - command: string, - args: string[] = ["--version"], -): Promise { - await runChecked(command, args, { timeoutMs: 30_000 }); -} - -async function packNpmPackage( - rootDir: string, - workDir: string, -): Promise { - const packDir = `${workDir}/packed`; - await Deno.mkdir(packDir, { recursive: true }); - const result = await runChecked("npm", [ - "pack", - "--pack-destination", - packDir, - ], { - cwd: `${rootDir}/npm`, - timeoutMs: 120_000, - }); - const tarball = result.stdout.split(/\r?\n/) - .map((line) => line.trim()) - .findLast((line) => line.endsWith(".tgz")); - - if (!tarball) { - throw new Error(`npm pack did not report a tarball:\n${result.stdout}`); - } - - return `${packDir}/${tarball}`; -} - -async function updateVeryfrontDependency( - projectDir: string, - tarballPath: string, -): Promise { - const packagePath = `${projectDir}/package.json`; - const pkg = JSON.parse(await Deno.readTextFile(packagePath)); - pkg.dependencies ??= {}; - pkg.dependencies.veryfront = `file:${tarballPath}`; - await Deno.writeTextFile(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); -} - -async function usePackedVeryfrontDenoTasks( - projectDir: string, - tarballPath: string, -): Promise { - const packagePath = `${projectDir}/package.json`; - const pkg = JSON.parse(await Deno.readTextFile(packagePath)); - delete pkg.dependencies?.veryfront; - await Deno.writeTextFile(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); - - const packedCliDir = `${projectDir}/.veryfront-packed-cli`; - await Deno.mkdir(packedCliDir, { recursive: true }); - await runChecked("tar", ["-xzf", tarballPath, "-C", packedCliDir], { - timeoutMs: 30_000, - }); - await runChecked("deno", ["install"], { - cwd: `${packedCliDir}/package`, - timeoutMs: 180_000, - }); - - const cliPath = JSON.stringify(`${packedCliDir}/package/esm/cli/main.js`); - const denoConfigPath = `${projectDir}/deno.json`; - const config = JSON.parse(await Deno.readTextFile(denoConfigPath)); - config.tasks ??= {}; - config.tasks.dev = `deno run -A ${cliPath} dev`; - config.tasks.build = `deno run -A ${cliPath} build`; - config.tasks.preview = `deno run -A ${cliPath} preview`; - await Deno.writeTextFile( - denoConfigPath, - `${JSON.stringify(config, null, 2)}\n`, - ); -} - -function assertCondition(condition: boolean, message: string): void { - if (!condition) { - throw new Error(message); - } -} - -function allocatePort(): number { - const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); - const port = (listener.addr as Deno.NetAddr).port; - listener.close(); - return port; -} - -async function waitForRoute(url: string, timeoutMs = 60_000): Promise { - const deadline = Date.now() + timeoutMs; - let lastError = ""; - - while (Date.now() < deadline) { - try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 1_000); - const response = await fetch(url, { signal: controller.signal }); - clearTimeout(timeout); - if (response.ok) { - await response.body?.cancel(); - return; - } - lastError = `HTTP ${response.status}`; - await response.body?.cancel(); - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - } - - await new Promise((resolve) => setTimeout(resolve, 500)); - } - - throw new Error( - `${url} did not become ready within ${timeoutMs}ms: ${lastError}`, - ); -} - async function verifyHttpRoute( url: string, expectedText: string[] = [], @@ -306,98 +125,6 @@ async function verifyHttpRoute( } } -async function collectStream( - stream: ReadableStream | null, - output: string[], -): Promise { - if (!stream) { - return; - } - - const reader = stream.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - return; - } - output.push(decoder.decode(value)); - } - } finally { - reader.releaseLock(); - } -} - -function startDevServer( - projectDir: string, - runtime: RuntimeName, - port: number, -): { - child: Deno.ChildProcess; - status: Promise; - stdout: string[]; - stderr: string[]; -} { - const { command, args } = getDevServerCommand(runtime, port); - const stdout: string[] = []; - const stderr: string[] = []; - const child = new Deno.Command(command, { - args, - cwd: projectDir, - env: { - LOG_FORMAT: "text", - NODE_ENV: "development", - REVALIDATION_PER_PROJECT_LIMIT: "0", - SSR_TRANSFORM_PER_PROJECT_LIMIT: "0", - VF_DISABLE_LRU_INTERVAL: "1", - }, - stdout: "piped", - stderr: "piped", - }).spawn(); - - void collectStream(child.stdout, stdout); - void collectStream(child.stderr, stderr); - - return { child, status: child.status, stdout, stderr }; -} - -export function getDevServerCommand( - runtime: RuntimeName, - port: number, -): { command: string; args: string[] } { - return { - command: runtime === "node" ? "npm" : runtime, - args: runtime === "deno" - ? ["task", "dev", "--port", String(port)] - : ["run", "dev", "--", "--port", String(port)], - }; -} - -async function stopDevServer(server: { - child: Deno.ChildProcess; - status: Promise; -}): Promise { - try { - server.child.kill("SIGTERM"); - } catch { - return; - } - - const exited = await Promise.race([ - server.status.then(() => true).catch(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), - ]); - - if (!exited) { - try { - server.child.kill("SIGKILL"); - } catch { - // The process may have exited between the timeout and SIGKILL. - } - await server.status.catch(() => {}); - } -} - function parseBrowserEnvelope(stdout: string, command: string): T { const envelope = JSON.parse(stdout) as BrowserEnvelope; if (!envelope.success) { @@ -474,83 +201,6 @@ async function verifyBrowserRoute( } } -async function scaffoldProject( - rootDir: string, - workDir: string, - tarballPath: string, - template: TemplateName, - runtime: RuntimeName, -): Promise { - const caseDir = `${workDir}/${runtime}-${template}`; - const projectName = `vf-${runtime}-${template}`; - await Deno.mkdir(caseDir, { recursive: true }); - await runChecked("npm", [ - "exec", - "--yes", - "--package", - tarballPath, - "--", - "veryfront", - "init", - projectName, - "--template", - template, - "--runtime", - runtime, - "--skip-install", - "--skip-env-prompt", - ], { - cwd: caseDir, - env: { - npm_config_cache: `${workDir}/npm-cache`, - npm_config_fund: "false", - npm_config_audit: "false", - }, - timeoutMs: 120_000, - }); - - const projectDir = `${caseDir}/${projectName}`; - if (runtime === "deno") { - await usePackedVeryfrontDenoTasks(projectDir, tarballPath); - } else { - await updateVeryfrontDependency(projectDir, tarballPath); - } - - if (rootDir.length === 0) { - throw new Error("Root directory could not be resolved"); - } - - return projectDir; -} - -async function installDependencies( - projectDir: string, - runtime: RuntimeName, - workDir: string, -): Promise { - if (runtime === "node") { - await runChecked("npm", ["install", "--no-audit", "--fund=false"], { - cwd: projectDir, - env: { npm_config_cache: `${workDir}/npm-cache` }, - timeoutMs: 180_000, - }); - return; - } - - if (runtime === "deno") { - await runChecked("deno", ["install"], { - cwd: projectDir, - timeoutMs: 180_000, - }); - return; - } - - await runChecked("bun", ["install"], { - cwd: projectDir, - timeoutMs: 180_000, - }); -} - async function verifyAgenticWorkflowDemo(rootUrl: string): Promise { const topic = `Runtime E2E ${crypto.randomUUID().slice(0, 8)}`; const startResponse = await fetch( @@ -608,7 +258,6 @@ async function verifyAgenticWorkflowDemo(rootUrl: string): Promise { } async function testCase( - rootDir: string, workDir: string, tarballPath: string, template: TemplateName, @@ -617,7 +266,6 @@ async function testCase( const label = `${runtime}/${template}`; console.log(`test ${label}: scaffold`); const projectDir = await scaffoldProject( - rootDir, workDir, tarballPath, template, @@ -652,6 +300,7 @@ async function testCase( await verifyAgenticWorkflowDemo(rootUrl); } } catch (error) { + await stopDevServer(server); const stdout = server.stdout.join("").trim(); const stderr = server.stderr.join("").trim(); throw new Error( @@ -706,7 +355,7 @@ async function main(): Promise { for (const template of templates) { for (const runtime of runtimes) { - await testCase(rootDir, workDir, tarballPath, template, runtime); + await testCase(workDir, tarballPath, template, runtime); } }