diff --git a/deno.json b/deno.json index ee1e633cc4..ab07e2290c 100644 --- a/deno.json +++ b/deno.json @@ -99,6 +99,8 @@ "./markdown": "./src/markdown/index.ts", "./mdx": "./src/mdx/index.ts", "./agent": "./src/agent/index.ts", + "./agent/executor-runtime": "./src/agent/hosted/executor-runtime-entrypoint.ts", + "./agent/managed-broker": "./src/agent/service/managed-broker.ts", "./agent/identity": "./src/agent/identity-contracts.ts", "./eval": "./src/eval/index.ts", "./metrics": "./src/metrics/public.ts", @@ -531,7 +533,7 @@ "lint:ci-typescript": "deno check --unstable-sloppy-imports --frozen --config=scripts/test.deno.json scripts/ci/npm-compatibility-artifact.ts scripts/ci/registry-release-integrity.ts scripts/ci/registry-release-integrity.test.ts scripts/ci/publish-npm-packages.test.ts tests/integration/ci/node-executor-coverage.test.ts tests/integration/ci/merge-quality-gate-workflow.test.ts tests/integration/ci/npm-compatibility-artifact-workflow.test.ts tests/integration/ci/npm-compatibility-artifact.test.ts tests/integration/ci/runtime-inference-critical-flow-packed-artifact.test.ts tests/integration/ci/registry-release-smoke.test.ts tests/integration/ci/registry-release-workflow.test.ts && deno fmt --check --line-width=100 --config=scripts/test.deno.json scripts/ci/npm-compatibility-artifact.ts scripts/ci/registry-release-integrity.ts scripts/ci/registry-release-integrity.test.ts tests/integration/ci/ && deno lint --config=scripts/test.deno.json scripts/test/coverage-node-executor.mjs scripts/ci/npm-compatibility-artifact.ts scripts/ci/registry-release-integrity.ts scripts/ci/registry-release-integrity.test.ts tests/integration/ci/", "fmt": "deno fmt src/ cli/ react/ templates/ && deno fmt --config=scripts/test.deno.json scripts/test/ scripts/build/dnt-compiler-options.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.ts scripts/build/dnt-polyfill.test.ts scripts/build/prepare-framework-sources.test.ts && deno fmt --config=scripts/codemods/deno.json scripts/codemods/", "fmt:check": "deno fmt --check src/ cli/ react/ templates/ && deno fmt --check --config=scripts/test.deno.json scripts/test/ scripts/build/dnt-compiler-options.ts scripts/build/dnt-jsx-runtime.test.ts scripts/build/dnt-meta-property-safety.ts scripts/build/dnt-meta-property-safety.test.ts scripts/build/dnt-polyfill.ts scripts/build/dnt-polyfill.test.ts scripts/build/prepare-framework-sources.test.ts && deno fmt --check --config=scripts/codemods/deno.json scripts/codemods/", - "typecheck": "deno task generate:manifests:check && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/html/hydration-script-builder/runtime/main.ts src/modules/index.ts src/proxy/main.ts src/react/components/ui/index.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/agent/service/route-export.check.ts src/eval/index.ts src/tool/index.ts src/workflow/index.ts src/prompt/index.ts src/resource/index.ts src/runs/index.ts src/run-events/index.ts src/mcp/index.ts src/provider/index.ts", + "typecheck": "deno task generate:manifests:check && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/html/hydration-script-builder/runtime/main.ts src/modules/index.ts src/proxy/main.ts src/react/components/ui/index.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/agent/service/route-export.check.ts src/agent/service/managed-broker.ts src/agent/hosted/executor-runtime-entrypoint.ts src/eval/index.ts src/tool/index.ts src/workflow/index.ts src/prompt/index.ts src/resource/index.ts src/runs/index.ts src/run-events/index.ts src/mcp/index.ts src/provider/index.ts", "verify": "deno task generate:manifests:check && deno task fmt:check && deno task lint && deno task lint:ci-typescript && deno task lint:style && deno task lint:chat-composability && deno task lint:chat-ratchets && deno task lint:esm-sh-codemod && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task lint:ban-test-only && deno task lint:sanitizer-baseline && deno task lint:skipped-tests && deno task lint:ban-zod && deno task lint:cwd-relative-test-reads && deno task lint:test-semantic-dispositions && deno task lint:anti-slop && deno task lint:render-mode-defaults && deno task lint:testing-front-door && deno task lint:core-deps && deno task lint:cross-runtime-jsr && deno task lint:dependency-boundaries && deno task lint:module-boundaries && deno task lint:extension-contracts && deno task lint:extension-capabilities && deno task docs:api-reference:check && deno task docs:errors:check && deno task docs:validate && deno task typecheck && deno task typecheck:consumer && deno task test && deno task test:scripts && deno task test:e2e:binary", "verify:quick": "deno task generate:manifests:check && deno task test:layout && deno task fmt:check && deno task lint && deno task lint:ci-typescript && deno task lint:style && deno task lint:chat-composability && deno task lint:chat-ratchets && deno task lint:esm-sh-codemod && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task lint:ban-test-only && deno task lint:sanitizer-baseline && deno task lint:skipped-tests && deno task lint:ban-zod && deno task lint:cwd-relative-test-reads && deno task lint:test-semantic-dispositions && deno task lint:anti-slop && deno task lint:render-mode-defaults && deno task lint:core-deps && deno task lint:cross-runtime-jsr && deno task lint:dependency-boundaries && deno task lint:module-boundaries && deno task lint:extension-contracts && deno task lint:extension-capabilities && deno task docs:api-reference:check && deno task docs:errors:check && deno task docs:validate && deno task typecheck", "typecheck:consumer": "deno run --allow-read --allow-run --allow-env --allow-write scripts/typecheck/run-consumer-typecheck.ts", @@ -608,6 +610,7 @@ "test:e2e:playwright": "deno run -A npm:playwright@1.60.0 test --config=tests/e2e/playwright.config.cjs", "test:e2e:playwright:install": "deno run -A npm:playwright@1.60.0 install --with-deps chromium", "test:e2e:rsc-browser": "deno task generate && deno run --config=scripts/test.deno.json --allow-read --allow-run=deno --allow-env scripts/test/run-deno-suite.ts --suite=e2e:rsc-browser", + "test:e2e:managed-broker": "deno run -A scripts/test/npm-install-smoke.ts --managed-broker-only", "test:e2e:binary": "deno task generate && deno run --config=scripts/test.deno.json --allow-read --allow-run=deno --allow-env scripts/test/run-deno-suite.ts --suite=e2e:binary", "test:e2e:binary:fresh": "deno task generate && VERYFRONT_BINARY_FRESH=1 deno run --config=scripts/test.deno.json --allow-read --allow-run=deno --allow-env scripts/test/run-deno-suite.ts --suite=e2e:binary", "test:e2e:templates": "deno run --allow-all scripts/test/template-runtime-e2e.ts", diff --git a/docs/api-reference/veryfront/agent.md b/docs/api-reference/veryfront/agent.md index 1f9e8e492c..fe1ab2ed27 100644 --- a/docs/api-reference/veryfront/agent.md +++ b/docs/api-reference/veryfront/agent.md @@ -1921,6 +1921,22 @@ Input delivered to a hosted agent-service detached execution callback. These import paths group focused functionality under this module. Each is a separate barrel; import only what you need. +### `veryfront/agent/executor-runtime` + +```ts +import { + initializeExecutorRuntimeContracts, + startExecutorRuntimeEntrypoint, +} from "veryfront/agent/executor-runtime"; +``` + +#### Functions + +| Name | Description | Source | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `initializeExecutorRuntimeContracts` | Install the fixed first-party contracts required before executor project imports. Concurrent and repeated startup preserves any already-registered trusted generation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/executor-runtime-contracts.ts) | +| `startExecutorRuntimeEntrypoint` | Dedicated executor entrypoint. The reviewed image launcher registers its first-party SchemaValidator, Bundler, ModuleLexer and SkillDocumentParserProvider before calling this function. The fixed image manifest is outside the project tree and is never selected by channel input. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/executor-runtime-entrypoint.ts) | + ### `veryfront/agent/identity` ```ts @@ -1964,3 +1980,58 @@ import { | `ProjectAgentRunSnapshot` | | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/identity-contracts.ts) | | `SourceProjectAgentExecutionIdentity` | | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/identity-contracts.ts) | | `SourceProjectAgentRunSnapshot` | | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/identity-contracts.ts) | + +### `veryfront/agent/managed-broker` + +Managed broker composition without project runtime or application imports. + +```ts +import { + connectExecutorTransport, + createHostedExecutorAllocatorClient, + createManagedBrokerHandler, +} from "veryfront/agent/managed-broker"; +``` + +#### Functions + +| Name | Description | Source | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `connectExecutorTransport` | Node-only TLS 1.3 PSK. No certificate fallback, session reuse, or reconnect. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/executor-node-transport.ts) | +| `createHostedExecutorAllocatorClient` | Trusted broker client for the operator's dedicated TLS endpoint. Each call reads the rotated Pod-bound token. No redirects, automatic POST retries, arbitrary headers, application credentials, or ambient gateway fallback. The returned promise retains raw token-read and socket ownership; the session supplies prompt cancellation notification separately. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/executor-allocator-client.ts) | +| `createManagedBrokerHandler` | Handle signed run invocations with configured detached or request-owned SSE responses. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-broker-handler.ts) | +| `createManagedBrokerPersistence` | Create exact-run API persistence callbacks while retaining credentials in the broker. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-broker-persistence.ts) | +| `createManagedBrokerProjectState` | Create broker-owned project steering. Refreshes show skills only when the broker-validated effective tool selection includes `load_skill`. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-broker-project-state.ts) | +| `createManagedExecutorBroker` | Compose an executor pool with authenticated installation and operation gates. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-executor-broker.ts) | +| `parseBrokerRuntimeAgentIngress` | Read and verify a signed invocation once before constructing executor-safe data. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `parseManagedAgUiAgentIngress` | Parse the trusted broker's direct request-owned AG-UI ingress. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-hosted-ingress.ts) | +| `parseManagedDurableAgentIngress` | Parse the trusted broker's direct canonical durable-run ingress. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-hosted-ingress.ts) | +| `startNodeManagedAgentBroker` | Bind managed routes and stop admission before joining all work during shutdown. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-node-broker.ts) | + +#### Classes + +| Name | Description | Source | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `BrokerIngressError` | Local HTTP-boundary errors intentionally avoid the application error registry: only a fixed code/status is exposed, never body, credential, or verifier diagnostics. These errors stay in the broker and are not executor-channel error contracts. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | + +#### Types + +| Name | Description | Source | +| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `BrokerIngressErrorCode` | Fixed, credential-free ingress failure identifiers. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `BrokerIngressScopeInput` | Signed identity and credentials supplied to the trusted scope verifier. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `BrokerRuntimeAgentExecutorInput` | Validated application data that can cross the executor channel. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `BrokerRuntimeAgentIngress` | Separate private authority and executor-safe invocation data. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `BrokerRuntimeAgentIngressOptions` | Broker-owned verification policy for one expected run and source. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `BrokerRuntimeAgentPrivateAuthority` | HTTP credentials and verified authority retained exclusively in the broker. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/broker-ingress.ts) | +| `ConnectExecutorTransportOptions` | | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/executor-node-transport.ts) | +| `ManagedAgentBrokerIngressAuthority` | Private parsed request and credentials available only to trusted broker preparation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-hosted-ingress.ts) | +| `ManagedAgentExecutorRequest` | Detached bounded application data, without HTTP objects or broker credentials. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-hosted-ingress.ts) | +| `ManagedAgentIngressResult` | Preserve the distinct durable and direct AG-UI ingress contracts. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-hosted-ingress.ts) | +| `ManagedBrokerOutput` | Acknowledging output writes and terminal finalization for a canonical run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-broker-persistence.ts) | +| `ManagedExecutorBrokerOptions` | Process admission and shutdown limits for a managed broker. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-executor-broker.ts) | +| `ManagedExecutorRuntime` | Prepared executor handle with broker-owned execution and retirement. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-executor-broker.ts) | +| `ManagedExecutorStarter` | Trusted executor admission boundary with actual settlement notification. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-broker-handler.ts) | +| `ManagedExecutorStartInput` | Trusted per-invocation source, model, tool, persistence, and state authority. Broker model limits, provider tools, and tool capabilities must not exceed the corresponding installed grant. Startup rejects mismatches before allocation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/managed-executor-broker.ts) | +| `ManagedNodeBrokerHandler` | Trusted broker route handler and optional retirement hook. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-node-broker.ts) | +| `ManagedNodeBrokerPool` | Broker admission and settlement lifecycle retained by the HTTP server. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/service/managed-node-broker.ts) | diff --git a/docs/guides/agent-service-runtime.md b/docs/guides/agent-service-runtime.md index ae92ab79bc..a47a4c65fa 100644 --- a/docs/guides/agent-service-runtime.md +++ b/docs/guides/agent-service-runtime.md @@ -50,12 +50,11 @@ trace hooks, or application-error reporters. The framework-owned `veryfront serve` runtime owns this setup on shared and managed dedicated servers. -The service captures request accessors and routing primitives before project -modules load. Routing and CORS checks use those captured operations so later -changes to shared web prototypes cannot inspect the run-event or inference -credentials on an incoming request. Import the framework service runtime before -loading project modules. Custom host route handlers still receive the original -request and remain responsible for authentication and credential handling. +The standalone service shares a process with the Agent code it loads. Use it +for trusted code. Captured request accessors protect specific ingress operations; +they do not provide process isolation for request bodies or credentials. +Custom host route handlers receive the original request and remain responsible +for authentication and credential handling. Dispatch visits the host route table and matched path segments by index so a replaced array iterator cannot inject a handler before host authentication. CORS allowlist membership, response header writes, and route path parsing also @@ -300,6 +299,23 @@ Services that use Veryfront Cloud project steering can reuse `fetchDefaultAgentServiceProjectSteering()` for the initial fetch and `createDefaultAgentServiceProjectSteeringRefresh()` for step-boundary refresh. +Managed brokers can use `createManagedBrokerProjectState()` from +`veryfront/agent/managed-broker` for project instructions and skill catalogs. +The executor sends its effective tool selection on each steering refresh. The +broker validates that selection against the installed grant before constructing +instructions, and includes skills only when `load_skill` remains available. +Refreshes without a tool selection omit the skill catalog. + +Managed broker ingress checks application strings and property names for known +credentials after URI decoding. Malformed escapes do not suppress checks on valid +encoded segments. Decoding is limited to 16 passes; ingress rejects strings that +still require decoding after that limit. Keep credentials out of executor data, +including encoded URLs and metadata. + +Closing the managed broker handler or aborting its service signal returns 503 +`BROKER_UNAVAILABLE` for pending admission. Request-only cancellation returns 499 +`BROKER_INGRESS_ABORTED`. + ## Keep inference authority separate Signed runtime invocations may include an optional @@ -418,6 +434,76 @@ A durable execution without authority bound to the expected run fails before provider dispatch. Token exchange failures are bounded, sanitized, and fail closed; callers must not retry by falling back to a user API token. +## Managed executor startup + +`startExecutorRuntimeEntrypoint` is available from +`veryfront/agent/executor-runtime`. It starts the executor side of a managed +broker/executor deployment on Node.js 22 or newer. Your trusted image launcher +calls `initializeExecutorRuntimeContracts()` from the same export to initialize +the first-party schema validator, bundler, module lexer, and Skill document +parser, then supplies the Operator allocation environment and image +manifest. Missing runtime contracts fail startup. + +The executor accepts one authenticated `runtime.install` message bound to its +allocation, invocation, generation, owner, and immutable source. Discovery and +runtime preparation remain unavailable until installation succeeds. The +installation carries runtime grants and capability IDs. Initial checkpoint +state uses a separate bounded stream so durable replay state can exceed the +installation message limit. + +The broker owns HTTP authentication, credentials, model and tool authorization, +and durable persistence. Executor facades call these capabilities through the +authenticated channel. Closing a runtime revokes its facades; admission remains +held until the original work and cleanup settle. This entrypoint requires the +broker and isolation infrastructure to be configured separately. + +Use `veryfront/agent/managed-broker` for the broker composition and signed +control-plane HTTP adapter. The broker installs invocation grants, describes +the selected agent, and prepares the executor before accepting a run. It keeps +model and tool execution unavailable during preparation. Configure detached +202 responses or request-owned SSE responses in trusted service configuration. +Signed, direct durable, and direct AG-UI ingress reject application strings or property +names containing a known broker credential, including credentials embedded in messages +or attachment URLs. +Detached runs require output persistence callbacks; their finalization remains +part of the session's owned work until all writes settle. + +Canonical runs must pass the persistence adapter's `bindSessionOwnedWork` +callback in `ManagedExecutorStartInput`. The broker binds it after reserving a +session and before installation or preparation. Scheduled, retry, and explicit +event-queue writes use that session owner. A persistence timeout can return +promptly while pool capacity stays reserved until the original write settles. + +`startNodeManagedAgentBroker` binds the signed stream, durable start, AG-UI, +and cancel/resume handlers to a Node server. Supply every handler, the broker +pool, and a readiness check explicitly. It preserves `/liveness` and +`/readiness`; shutdown stops admission before waiting for handlers and broker +work to retire. This server adapter does not configure product policy, +registration, credentials, or executor images. + +Managed run routes return HTTP 400 with `BROKER_INGRESS_TARGET_MISMATCH` when +a run ID contains malformed URL encoding or fails the canonical run ID schema +after decoding. Run IDs contain 1 to 128 ASCII letters, digits, underscores, +or hyphens. Valid encoded run IDs are decoded before the handler receives them. +Signed stream requests must sign the original encoded request path. + +To verify the packaged broker and executor locally, use Node.js 22.3.0 or newer: + +```bash +deno task build:npm +deno task test:e2e:managed-broker +``` + +This suite installs the built packages and exercises signed HTTP ingress, a +separate executor over TLS, model and tool calls, SSE and detached responses, +executor termination, client cancellation, delayed terminal persistence, and +steering refresh with provider-native tools. Empty replay snapshots remain valid +when provider replay is disabled. +Project-controlled hooks use synthetic credential canaries with positive +controls. The npm smoke jobs run the suite on the minimum supported Node version +and the current CI version. These checks use local synthetic services. Verify +the deployed artifact and isolation configuration separately before traffic cutover. + ## Verify it worked Start the service entrypoint and call the run route directly. The default @@ -434,3 +520,22 @@ A working service streams AG-UI events back. If Veryfront Cloud registration is enabled, the service should also appear in the cloud dashboard's agent service list after the first heartbeat (`VERYFRONT_AGENT_SERVICE_HEARTBEAT_INTERVAL_MS`). + +Broker model output limits and provider-tool descriptors must stay within the installed model grant. +Each broker tool capability must also stay within the installed tool allowlist. Startup rejects broader +broker authority before allocating an executor. Preparation uses the narrower broker model output +limits and provider-tool list, so its default model requests fit the broker policy. Anthropic thinking +with an additive token budget reserves that budget from the total allowance before preparation chooses +the completion limit. Adaptive thinking uses the total allowance without an additive reservation. +Preparation rejects an explicit completion limit that exceeds the remainder. Source IDs must +belong to the installed host-facade or remote-source grants. Owner-scoped tool selectors use the same +canonical names for capability checks and steering refreshes. + +Trusted ingress must provide `tools.catalog` with the complete tool inventory and ownership metadata, +including project-local tools that have no broker capability. The broker resolves short selectors to +owned tools first, then validates source capabilities against those exact IDs. Preparation and steering +refreshes receive the same resolved grant. A shadowed global tool does not gain authority from an owned +tool's short selector. The catalog must come from trusted source metadata before executor discovery. +For selected host tools, the broker includes the catalog's owner and short-name mapping in the +validated installation. Rebuilt executor facades retain that mapping, so an agent's short selector +continues to select the same canonical tool. Remote source listings do not supply ownership authority. diff --git a/scripts/test/coverage-node-executor.mjs b/scripts/test/coverage-node-executor.mjs index 33a355a42a..eabb2c98b2 100644 --- a/scripts/test/coverage-node-executor.mjs +++ b/scripts/test/coverage-node-executor.mjs @@ -10,6 +10,7 @@ const ROOT = fileURLToPath(new URL("../../", import.meta.url)); const SOURCE_FILES = [ "src/agent/hosted/executor-allocator-client.ts", "src/agent/hosted/executor-node-bootstrap.ts", + "src/agent/hosted/executor-runtime-entrypoint.ts", "src/agent/hosted/executor-node-transport.ts", "src/security/http/native-header-processing.ts", "src/security/http/native-request-processing.ts", @@ -69,7 +70,11 @@ export async function validateNativeCoverage( if (match) record.lines.set(Number(match[1]), Number(match[2])); } else if (record && line.startsWith("FN:")) { const match = /^FN:(\d+),(.+)$/.exec(line); - if (match) record.functions.set(match[2], Number(match[1])); + if (match) { + const positions = record.functions.get(match[2]) ?? []; + positions.push(Number(match[1])); + record.functions.set(match[2], positions); + } } else if (record && line === "end_of_record") { records.set(record.source, record); record = undefined; @@ -100,11 +105,20 @@ export async function validateNativeCoverage( first--; } first = Math.max(1, first - 1); - const mappedLine = anchor && entry.functions.get(anchor.name); + const functionEndOffset = anchor && + lines.slice(anchor.line).findIndex((line) => line === "}"); + const functionEnd = functionEndOffset === -1 || !anchor + ? undefined + : anchor.line + functionEndOffset + 1; + const mappedLines = anchor && entry.functions.get(anchor.name); + const hasOriginalFunctionPosition = mappedLines?.some((line) => + functionEnd !== undefined && line >= first && line <= functionEnd + ) === true; if ( - !anchor || mappedLine === undefined || mappedLine < first || - mappedLine > anchor.line || - [...entry.lines.keys()].some((line) => line < 1 || line > lines.length) + !anchor || !hasOriginalFunctionPosition || + [...entry.lines.keys()].some((line) => + line < 1 || line > lines.length + ) ) { throw new Error( `Native coverage is not mapped to original source: ${basename(source)}`, diff --git a/scripts/test/npm-install-smoke.ts b/scripts/test/npm-install-smoke.ts index 65f22810ed..93b2358f52 100644 --- a/scripts/test/npm-install-smoke.ts +++ b/scripts/test/npm-install-smoke.ts @@ -21,6 +21,9 @@ * 9. a packed agent workflow reaches a non-responsive provider, respects * its configured deadline, persists failure, and leaves the server * healthy + * 10. the managed broker executes through a separate executor process, + * retains persistence during shutdown, and keeps synthetic credentials + * out of project-controlled hooks * * The runtime under test stays the packed npm artifact under the ambient Node * version: this orchestrator only spawns `npm`, `node`, and `deno eval` @@ -90,6 +93,7 @@ async function run( options: { cwd?: string; env?: Record; + clearEnv?: boolean; timeoutMs: number; }, ): Promise { @@ -100,6 +104,7 @@ async function run( args, cwd: options.cwd, env: options.env, + clearEnv: options.clearEnv, stdin: "null", stdout: "piped", stderr: "piped", @@ -124,7 +129,12 @@ async function runChecked( step: string, command: string, args: string[], - options: { cwd?: string; env?: Record; timeoutMs: number }, + options: { + cwd?: string; + env?: Record; + clearEnv?: boolean; + timeoutMs: number; + }, ): Promise { const result = await run(command, args, options); if (result.code !== 0) { @@ -1177,6 +1187,33 @@ async function checkWorkflowTimeout( } } +async function checkManagedBroker(workDir: string): Promise { + console.log("== packed managed broker: process boundary and settlement"); + const fixtureDir = `${workDir}/managed-broker`; + await Deno.mkdir(fixtureDir, { recursive: true }); + for (const name of ["journey.mjs", "executor.mjs", "project-hooks.mjs"]) { + await Deno.copyFile( + `${ROOT_DIR}/tests/e2e/agent/managed-broker/${name}`, + `${fixtureDir}/${name}`, + ); + } + const result = await runChecked("managed broker journey", "node", [ + "--test", + "--test-concurrency=1", + `${fixtureDir}/journey.mjs`, + ], { + cwd: workDir, + clearEnv: true, + env: { + PATH: Deno.env.get("PATH") ?? "", + NODE_ENV: "production", + VF_DISABLE_LRU_INTERVAL: "1", + }, + timeoutMs: 480_000, + }); + console.log(result.stdout); +} + async function runSmoke(workDir: string): Promise { let devServer: DevServer | undefined; const shutdown = async () => { @@ -1209,6 +1246,11 @@ async function runSmoke(workDir: string): Promise { }); await npmInstall(workDir, plan, plan.rootInstallSpecs); + if (Deno.args.includes("--managed-broker-only")) { + await checkManagedBroker(workDir); + return; + } + await checkRootInstall(workDir); await checkOptionalPeer(workDir); await checkMissingExtension(workDir); @@ -1225,6 +1267,7 @@ async function runSmoke(workDir: string): Promise { await checkWorkflowTimeout(devServer, devUrl, csrfToken); await shutdown(); + await checkManagedBroker(workDir); console.log("npm install smoke: all checks passed"); } finally { for (const [signal, handler] of signalHandlers) { diff --git a/src/agent/conversation/durable-contracts.ts b/src/agent/conversation/durable-contracts.ts index 407006540b..6fe4e0efa6 100644 --- a/src/agent/conversation/durable-contracts.ts +++ b/src/agent/conversation/durable-contracts.ts @@ -452,4 +452,6 @@ export interface FinalizeConversationAgentRunInput { finishReason?: string; terminalErrorCode?: string | null; terminalErrorMessage?: string | null; + /** Explicit trusted-host transport for broker-owned finalization. */ + fetch?: typeof globalThis.fetch; } diff --git a/src/agent/conversation/durable.ts b/src/agent/conversation/durable.ts index 6ad5e3a588..e3996074d1 100644 --- a/src/agent/conversation/durable.ts +++ b/src/agent/conversation/durable.ts @@ -1398,5 +1398,6 @@ export async function finalizeConversationAgentRun( }, responseSchema: CompleteConversationRunResponseSchema, operation: "Complete canonical durable run", + fetch: input.fetch, }); } diff --git a/src/agent/conversation/hosted-terminal.ts b/src/agent/conversation/hosted-terminal.ts index 8194ad3cb5..343e9cc75c 100644 --- a/src/agent/conversation/hosted-terminal.ts +++ b/src/agent/conversation/hosted-terminal.ts @@ -136,6 +136,8 @@ export interface CreateConversationHostedTerminalAdapterOptions { fallbackModelId: string; resolveProvider: (modelId: string) => string; onTerminalState?: (terminalState: HostedLifecycleTerminalState) => Promise | void; + /** Explicit trusted-host transport for durable terminal persistence. */ + fetch?: typeof globalThis.fetch; } /** Public API contract for conversation hosted terminal adapter. */ @@ -245,6 +247,7 @@ export function createConversationHostedTerminalAdapter( ), terminalErrorCode: terminalState.terminalErrorCode, terminalErrorMessage: terminalState.terminalErrorMessage, + fetch: options.fetch, }); } catch (error) { // Allow a later dispatch to retry; keeping the flag set on failure would diff --git a/src/agent/conversation/run-chunk-mirror.test.ts b/src/agent/conversation/run-chunk-mirror.test.ts index 4d24714f15..45a1168945 100644 --- a/src/agent/conversation/run-chunk-mirror.test.ts +++ b/src/agent/conversation/run-chunk-mirror.test.ts @@ -434,6 +434,42 @@ describe("agent/conversation-run-chunk-mirror", () => { } }); + it("forwards hosted queue flush ownership before an automatic append request", async () => { + const calls: string[] = []; + const mirror = createHostedConversationRunChunkMirror({ + authToken: "token", + apiUrl: "https://api.example.test", + conversationId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + latestEventId: 0, + batchSize: 1, + runQueueFlush: async (operation) => { + calls.push("owner"); + return await operation(); + }, + fetch: () => { + calls.push("fetch"); + return Promise.resolve(Response.json({ + latest_event_id: 1, + latest_external_event_sequence: 1, + appended_count: 1, + run: { + run_id: "run-1", + conversation_id: "11111111-1111-4111-8111-111111111111", + latest_event_id: 1, + latest_external_event_sequence: 1, + }, + })); + }, + }); + + await mirror.appendEvents([{ type: "TEXT_MESSAGE_CONTENT", delta: "owned" }]); + await mirror.flush(); + + assertEquals(calls, ["owner", "fetch"]); + mirror.dispose(); + }); + // VERYFRONT-AGENT-3: every retry_scheduled flush logged at error level, so a // degraded append endpoint emitted a Sentry error per ~5s retry per run. The // per-attempt log must stay at warn and escalate to error only once the diff --git a/src/agent/conversation/run-chunk-mirror.ts b/src/agent/conversation/run-chunk-mirror.ts index 29a40ad68f..14120c2718 100644 --- a/src/agent/conversation/run-chunk-mirror.ts +++ b/src/agent/conversation/run-chunk-mirror.ts @@ -10,6 +10,7 @@ import { type ConversationRunMirrorRetryScheduledState, type ConversationRunMirrorSnapshot, type ConversationRunMirrorStoppedState, + type ConversationRunQueueFlush, createConversationRunMirror, } from "./run-mirror.ts"; import { @@ -71,6 +72,7 @@ interface ConversationRunChunkMirrorSharedOptions { onHighBacklog?: (state: ConversationRunMirrorHighBacklogState) => Promise | void; onRetryScheduled?: (state: ConversationRunMirrorRetryScheduledState) => Promise | void; onStopped?: (state: ConversationRunMirrorStoppedState) => Promise | void; + runQueueFlush?: ConversationRunQueueFlush; prepareChunkEvents?: ( input: ConversationRunChunkMirrorPrepareChunkEventsInput, ) => Promise | ConversationRunEvent[]; @@ -135,6 +137,7 @@ export interface HostedConversationRunChunkMirrorOptions { batchSize?: number; highBacklogEventCount?: number; instrumentation?: HostedConversationRunChunkMirrorInstrumentation; + runQueueFlush?: ConversationRunQueueFlush; /** Explicit host-owned transport for trusted runtime composition and tests. */ fetch?: typeof globalThis.fetch; } @@ -188,6 +191,7 @@ export function createConversationRunChunkMirror( ...(input.onHighBacklog ? { onHighBacklog: input.onHighBacklog } : {}), ...(input.onRetryScheduled ? { onRetryScheduled: input.onRetryScheduled } : {}), ...(input.onStopped ? { onStopped: input.onStopped } : {}), + ...(input.runQueueFlush ? { runQueueFlush: input.runQueueFlush } : {}), }); return { @@ -418,6 +422,7 @@ export function createHostedConversationRunChunkMirror( immediateFlushEventCount: batchSize, highBacklogEventCount, fetch: input.fetch, + ...(input.runQueueFlush ? { runQueueFlush: input.runQueueFlush } : {}), prepareChunkEvents: ({ chunk, defaultPrepare }) => runHostedChunkMirrorTrace(input.instrumentation, "durable.mirrorChunk", async () => { const events = defaultPrepare(); diff --git a/src/agent/conversation/run-mirror.test.ts b/src/agent/conversation/run-mirror.test.ts index a6a6d60253..2323c89c19 100644 --- a/src/agent/conversation/run-mirror.test.ts +++ b/src/agent/conversation/run-mirror.test.ts @@ -200,6 +200,100 @@ describe("agent/conversation-run-mirror", () => { assertEquals(mirror.getSnapshot().pendingEventCount, 0); }); + it("runs immediate, retry, and explicit queue flushes inside the owner hook", async () => { + using time = new FakeTime(); + const calls: string[] = []; + let flushCalls = 0; + const controller = createMockQueueController({ + flushImpl: async () => { + calls.push("controller"); + flushCalls += 1; + if (flushCalls === 1) { + return { + outcome: "retry_scheduled" as const, + latestEventId: 0, + latestExternalEventSequence: 0, + pendingEventCount: 1, + consecutiveFailures: 1, + disabled: false, + errorMessage: "retry", + }; + } + return { + outcome: "flushed" as const, + latestEventId: flushCalls, + latestExternalEventSequence: flushCalls, + pendingEventCount: 0, + consecutiveFailures: 0, + disabled: false, + }; + }, + }); + const mirror = createConversationRunMirror({ + queueController: controller, + immediateFlushEventCount: 1, + getRetryDelayMs: () => 25, + runQueueFlush: async (operation) => { + calls.push("owner"); + return await operation(); + }, + }); + + mirror.enqueue([{ id: "immediate" }]); + await time.tickAsync(0); + await time.tickAsync(25); + mirror.enqueue([{ id: "explicit" }]); + await mirror.flush(); + + assertEquals(calls, [ + "owner", + "controller", + "owner", + "controller", + "owner", + "controller", + ]); + mirror.dispose(); + }); + + it("does not enter a delayed queue flush after its owner closes", async () => { + using time = new FakeTime(); + let active = true; + let ownerCalls = 0; + let controllerCalls = 0; + const controller = createMockQueueController({ + flushImpl: async () => { + controllerCalls += 1; + return { + outcome: "flushed" as const, + latestEventId: 1, + latestExternalEventSequence: 1, + pendingEventCount: 0, + consecutiveFailures: 0, + disabled: false, + }; + }, + }); + const mirror = createConversationRunMirror({ + queueController: controller, + immediateFlushEventCount: 2, + flushDelayMs: 50, + runQueueFlush: async (operation) => { + ownerCalls += 1; + if (!active) throw new Error("owner closed"); + return await operation(); + }, + }); + + mirror.enqueue([{ id: "delayed" }]); + active = false; + await time.tickAsync(50); + + assertEquals(ownerCalls, 1); + assertEquals(controllerCalls, 0); + mirror.dispose(); + }); + it("rejects an explicit flush when a controller failure escaped the queue", async () => { const controller = createMockQueueController({ pendingEvents: [], diff --git a/src/agent/conversation/run-mirror.ts b/src/agent/conversation/run-mirror.ts index d6c46c8546..41a4301981 100644 --- a/src/agent/conversation/run-mirror.ts +++ b/src/agent/conversation/run-mirror.ts @@ -57,6 +57,9 @@ export interface ConversationRunMirrorHighBacklogState { threshold: number; } +/** Own one underlying queue flush for its complete asynchronous lifetime. */ +export type ConversationRunQueueFlush = (operation: () => Promise) => Promise; + /** Public API contract for conversation run mirror. */ export interface ConversationRunMirror { enqueue(events: unknown[]): void; @@ -101,6 +104,7 @@ export function createConversationRunMirror(input: { onHighBacklog?: (state: ConversationRunMirrorHighBacklogState) => Promise | void; onRetryScheduled?: (state: ConversationRunMirrorRetryScheduledState) => Promise | void; onStopped?: (state: ConversationRunMirrorStoppedState) => Promise | void; + runQueueFlush?: ConversationRunQueueFlush; }): ConversationRunMirror { const flushDelayMs = input.flushDelayMs ?? DEFAULT_FLUSH_DELAY_MS; const getRetryDelayMs = input.getRetryDelayMs ?? getDefaultRetryDelayMs; @@ -209,9 +213,11 @@ export function createConversationRunMirror(input: { async function runFlushLoop(abortSignal?: AbortSignal): Promise { emitHighBacklogIfNeeded(); - const flushed = await input.queueController.flush({ - abortSignal: abortSignal ?? lifecycleAbortController.signal, - }); + const flush = () => + input.queueController.flush({ + abortSignal: abortSignal ?? lifecycleAbortController.signal, + }); + const flushed = await (input.runQueueFlush ? input.runQueueFlush(flush) : flush()); escapedFlushFailures = 0; escapedFlushError = null; diff --git a/src/agent/hosted/executor-allocator-client.ts b/src/agent/hosted/executor-allocator-client.ts index 4e188ef695..cbb56a04d9 100644 --- a/src/agent/hosted/executor-allocator-client.ts +++ b/src/agent/hosted/executor-allocator-client.ts @@ -4,6 +4,7 @@ import { Buffer } from "node:buffer"; import { lookup } from "node:dns/promises"; import { request as httpsRequest } from "node:https"; import process from "node:process"; +import { isNodeRuntime } from "#veryfront/platform/compat/runtime.ts"; import type { HostedExecutorAllocatorClient } from "#veryfront/agent/hosted/executor-session.ts"; import { getHostedExecutorAllocationRequestSchema, @@ -54,7 +55,7 @@ export function createHostedExecutorAllocatorClient(options: { timeoutMs?: number; }): HostedExecutorAllocatorClient { if ( - "Deno" in globalThis || "Bun" in globalThis || process.release.name !== "node" || + !isNodeRuntime() || process.release.name !== "node" || Number(process.versions.node.split(".")[0]) < 22 ) { throw new Error("Executor allocator client requires Node.js 22 or newer"); diff --git a/src/agent/hosted/executor-checkpoint-state.test.ts b/src/agent/hosted/executor-checkpoint-state.test.ts new file mode 100644 index 0000000000..3691a7ac4b --- /dev/null +++ b/src/agent/hosted/executor-checkpoint-state.test.ts @@ -0,0 +1,153 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; +import type { JsonValue } from "#veryfront/schemas/index.ts"; +import type { ProviderReplayCheckpoint } from "../runtime/provider-replay.ts"; +import { createExecutorChannel } from "../executor/channel.ts"; +import { + copyExecutorReplayCheckpoints, + createExecutorCheckpointStateOperations, + executorInitialCheckpointsOperation, + readExecutorInitialCheckpoints, +} from "./executor-checkpoint-state.ts"; + +const binding = { + allocationId: "checkpoint-allocation", + invocationId: "checkpoint-invocation", + generation: 1, +}; +function checkpoint(id: string, size: number): ProviderReplayCheckpoint { + return { + version: 1, + messageId: id, + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "x".repeat(size) }, + }], + providerBlockPositions: [0], + totalPartCount: 1, + elapsedMs: 12.5, + }; +} + +it("streams a checkpoint delivery larger than one channel frame and retains every anchor", async () => { + const checkpoints = Array.from( + { length: 6 }, + (_, index) => checkpoint(`message-${index}`, 200_000), + ); + const operations = createExecutorCheckpointStateOperations({ + expectedBinding: binding, + capabilityIds: { providerReplayCheckpoint: "replay" }, + initialProviderReplayCheckpoints: checkpoints, + }); + const toBroker = new TransformStream(); + const toExecutor = new TransformStream(); + const broker = createExecutorChannel({ + binding, + operations, + transport: { readable: toBroker.readable, writable: toExecutor.writable }, + }); + const executor = createExecutorChannel({ + binding, + transport: { readable: toExecutor.readable, writable: toBroker.writable }, + }); + try { + const restored = await readExecutorInitialCheckpoints({ + channel: executor, + capabilityIds: { providerReplayCheckpoint: "replay" }, + }); + assertEquals( + restored.initialProviderReplayCheckpoints?.map((value) => value.messageId), + checkpoints.map((value) => value.messageId), + ); + assertEquals( + copyExecutorReplayCheckpoints(restored.initialProviderReplayCheckpoints!).length, + 6, + ); + await assertRejects(() => + readExecutorInitialCheckpoints({ + channel: executor, + capabilityIds: { providerReplayCheckpoint: "replay" }, + }) + ); + } finally { + broker.close(); + executor.close(); + await Promise.all([broker.settled, executor.settled]); + } +}); + +it("rejects wrong binding and capability before consuming a snapshot grant", async () => { + const operation = createExecutorCheckpointStateOperations({ + expectedBinding: binding, + capabilityIds: { providerReplayCheckpoint: "replay" }, + initialProviderReplayCheckpoints: [checkpoint("message", 10)], + }).get(executorInitialCheckpointsOperation)!; + if (operation.mode !== "stream") throw new Error("Missing fixture stream"); + const handle = operation.handle; + const context = { binding, signal: new AbortController().signal, deadline: Date.now() + 10_000 }; + async function consume(value: JsonValue, bound = binding) { + const frames: JsonValue[] = []; + for await (const frame of handle(value, { ...context, binding: bound })) { + frames.push(frame); + } + return frames; + } + await assertRejects(() => consume({ kind: "provider-replay", capabilityId: "other" })); + await assertRejects(() => + consume({ kind: "provider-replay", capabilityId: "replay" }, { ...binding, generation: 2 }) + ); + await assertRejects(() => + consume({ kind: "provider-replay", capabilityId: "replay", runId: "other" }) + ); + assertEquals((await consume({ kind: "provider-replay", capabilityId: "replay" })).length, 2); + await assertRejects(() => consume({ kind: "provider-replay", capabilityId: "replay" })); +}); + +it("accepts an empty replay snapshot without granting replay access", async () => { + for (const capabilityIds of [{}, { toolExposureCheckpoint: "tools" }]) { + const operations = createExecutorCheckpointStateOperations({ + expectedBinding: binding, + capabilityIds, + initialProviderReplayCheckpoints: [], + }); + assertEquals(operations.size, capabilityIds.toolExposureCheckpoint ? 1 : 0); + const operation = operations.get(executorInitialCheckpointsOperation); + if (capabilityIds.toolExposureCheckpoint) { + assert(operation?.mode === "stream"); + await assertRejects( + async () => { + for await ( + const _frame of operation.handle({ kind: "provider-replay", capabilityId: "tools" }, { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 10_000, + }) + ) { + throw new Error("Replay access was not granted"); + } + }, + TypeError, + "Executor checkpoint state is not authorized", + ); + } + } +}); + +it("rejects duplicate anchors and missing grants instead of silently dropping state", () => { + assertThrows(() => copyExecutorReplayCheckpoints([checkpoint("same", 1), checkpoint("same", 2)])); + assertThrows(() => + copyExecutorReplayCheckpoints( + Array.from({ length: 101 }, (_, index) => checkpoint(String(index), 1)), + ) + ); + assertThrows(() => + createExecutorCheckpointStateOperations({ + expectedBinding: binding, + capabilityIds: {}, + initialProviderReplayCheckpoints: [checkpoint("message", 1)], + }) + ); +}); diff --git a/src/agent/hosted/executor-checkpoint-state.ts b/src/agent/hosted/executor-checkpoint-state.ts new file mode 100644 index 0000000000..bfeac80983 --- /dev/null +++ b/src/agent/hosted/executor-checkpoint-state.ts @@ -0,0 +1,156 @@ +import type { ExecutorChannel, ExecutorOperation } from "../executor/channel.ts"; +import { type ExecutorBinding, getExecutorBindingSchema } from "../executor/protocol.ts"; +import { defineSchema, type JsonValue } from "#veryfront/schemas/index.ts"; +import type { ToolExposureCheckpoint } from "../runtime/tool-exposure.ts"; +import { + parseServerResolvedProviderReplayCheckpoints, + type ProviderReplayCheckpoint, +} from "../runtime/provider-replay.ts"; +import { + type ExecutorPersistenceCapabilityIds, + executorPersistenceJson, + getExecutorPersistenceCapabilityIdsSchema, + getExecutorProviderReplayCheckpointSchema, + getExecutorToolExposureCheckpointSchema, + parseExecutorPersistenceData, +} from "./executor-persistence-schema.ts"; + +export const executorInitialCheckpointsOperation = "persistence.initial-checkpoints"; +const MAX_CHECKPOINTS = 100; +const getRequestSchema = defineSchema((v) => + v.object({ + capabilityId: v.string().min(1).max(128), + kind: v.enum(["tool-exposure", "provider-replay"] as const), + }).strict() +); +const getFrameSchema = defineSchema((v) => + v.discriminatedUnion("type", [ + v.object({ + type: v.literal("tool-exposure"), + checkpoint: getExecutorToolExposureCheckpointSchema(), + }).strict(), + v.object({ + type: v.literal("provider-replay"), + checkpoint: getExecutorProviderReplayCheckpointSchema(), + }).strict(), + v.object({ type: v.literal("complete") }).strict(), + ]) +); + +export interface ExecutorInitialCheckpointState { + initialToolExposureCheckpoint?: ToolExposureCheckpoint; + initialProviderReplayCheckpoints?: readonly ProviderReplayCheckpoint[]; +} + +/** Bound each checkpoint independently, matching the write path and canonical delivery count. */ +export function copyExecutorReplayCheckpoints( + value: readonly ProviderReplayCheckpoint[], +): ProviderReplayCheckpoint[] { + if (!Array.isArray(value) || value.length > MAX_CHECKPOINTS) { + throw new TypeError("Invalid executor checkpoint state"); + } + return parseServerResolvedProviderReplayCheckpoints( + value.map((checkpoint) => + parseExecutorPersistenceData( + getExecutorProviderReplayCheckpointSchema(), + executorPersistenceJson(checkpoint), + ) + ), + ); +} + +/** Each granted snapshot can be read once. State is never placed in runtime.install. */ +export function createExecutorCheckpointStateOperations( + options: ExecutorInitialCheckpointState & { + expectedBinding: ExecutorBinding; + capabilityIds: ExecutorPersistenceCapabilityIds; + }, +): ReadonlyMap { + const ids = getExecutorPersistenceCapabilityIdsSchema().parse(options.capabilityIds); + const binding = Object.freeze(getExecutorBindingSchema().parse(options.expectedBinding)); + if ( + options.initialToolExposureCheckpoint && !ids.toolExposureCheckpoint || + (options.initialProviderReplayCheckpoints?.length ?? 0) > 0 && !ids.providerReplayCheckpoint + ) throw new TypeError("Executor checkpoint state is not granted"); + const tool = options.initialToolExposureCheckpoint === undefined + ? undefined + : parseExecutorPersistenceData( + getExecutorToolExposureCheckpointSchema(), + executorPersistenceJson(options.initialToolExposureCheckpoint), + ); + const provider = copyExecutorReplayCheckpoints(options.initialProviderReplayCheckpoints ?? []); + const read = new Set(); + if (!ids.toolExposureCheckpoint && !ids.providerReplayCheckpoint) return new Map(); + return new Map([[executorInitialCheckpointsOperation, { + mode: "stream", + async *handle(value, context): AsyncGenerator { + const request = parseExecutorPersistenceData(getRequestSchema(), value); + context.signal.throwIfAborted(); + const expected = request.kind === "tool-exposure" + ? ids.toolExposureCheckpoint + : ids.providerReplayCheckpoint; + if ( + !expected || request.capabilityId !== expected || read.has(request.kind) || + context.binding.allocationId !== binding.allocationId || + context.binding.generation !== binding.generation || + context.binding.invocationId !== binding.invocationId || context.deadline <= Date.now() + ) throw new TypeError("Executor checkpoint state is not authorized"); + read.add(request.kind); + if (request.kind === "tool-exposure" && tool) { + yield executorPersistenceJson({ type: "tool-exposure", checkpoint: tool }); + } + if (request.kind === "provider-replay") { + for (const checkpoint of provider) { + context.signal.throwIfAborted(); + if (context.deadline <= Date.now()) { + throw new TypeError("Executor checkpoint state expired"); + } + yield executorPersistenceJson({ type: "provider-replay", checkpoint }); + } + } + context.signal.throwIfAborted(); + yield { type: "complete" }; + }, + }]]); +} + +export async function readExecutorInitialCheckpoints(options: { + channel: ExecutorChannel; + capabilityIds: ExecutorPersistenceCapabilityIds; + signal?: AbortSignal; +}): Promise { + const state: ExecutorInitialCheckpointState = {}; + for ( + const [kind, capabilityId] of [ + ["tool-exposure", options.capabilityIds.toolExposureCheckpoint], + ["provider-replay", options.capabilityIds.providerReplayCheckpoint], + ] as const + ) { + if (!capabilityId) continue; + let complete = false; + let count = 0; + const provider: ProviderReplayCheckpoint[] = []; + const frames = options.channel.stream(executorInitialCheckpointsOperation, { + capabilityId, + kind, + }, { signal: options.signal }); + for await (const value of frames) { + const frame = parseExecutorPersistenceData(getFrameSchema(), value); + if (complete) throw new TypeError("Invalid executor checkpoint completion"); + if (frame.type === "complete") { + complete = true; + continue; + } + if (frame.type !== kind || ++count > (kind === "tool-exposure" ? 1 : MAX_CHECKPOINTS)) { + throw new TypeError("Invalid executor checkpoint state"); + } + if (frame.type === "tool-exposure") state.initialToolExposureCheckpoint = frame.checkpoint; + else provider.push(frame.checkpoint); + } + if (!complete) throw new TypeError("Missing executor checkpoint completion"); + if (kind === "provider-replay") { + state.initialProviderReplayCheckpoints = copyExecutorReplayCheckpoints(provider); + } + } + return state; +} diff --git a/src/agent/hosted/executor-model-bridge.test.ts b/src/agent/hosted/executor-model-bridge.test.ts index a14d592453..c639557901 100644 --- a/src/agent/hosted/executor-model-bridge.test.ts +++ b/src/agent/hosted/executor-model-bridge.test.ts @@ -653,6 +653,32 @@ describe("executor managed model bridge", () => { } }); + it("contains provider stream errors during abort without an uncaught event rejection", async () => { + const channels = await connected(stubModel({ + doStream: ({ abortSignal }) => + Promise.resolve({ + stream: new ReadableStream({ + start(controller) { + abortSignal!.addEventListener("abort", () => { + controller.error(new Error("Synthetic provider abort failure")); + }, { once: true }); + }, + }), + }), + })); + try { + const controller = new AbortController(); + const { stream } = await channels.proxy.doStream({ prompt, abortSignal: controller.signal }); + controller.abort(); + await assertRejects(() => stream.getReader().read(), Error, "cancelled"); + // Node reports rejected promises returned by EventTarget listeners as + // uncaught exceptions, even when the original promise has a catch handler. + await tick(); + } finally { + await channels.close(); + } + }); + it("retains admission until asynchronous provider stream cleanup settles", async () => { const cleanup = Promise.withResolvers(); const cancelStarted = Promise.withResolvers(); diff --git a/src/agent/hosted/executor-model-bridge.ts b/src/agent/hosted/executor-model-bridge.ts index b8b1c37777..53d29a0051 100644 --- a/src/agent/hosted/executor-model-bridge.ts +++ b/src/agent/hosted/executor-model-bridge.ts @@ -227,7 +227,12 @@ export function createExecutorModelBroker(options: { } return cancellation; }; - context.signal.addEventListener("abort", cancel, { once: true }); + // Node EventTarget reports a rejected promise returned by a listener + // as an uncaught exception. Cleanup is observed and joined separately. + const onAbort = () => { + void cancel(); + }; + context.signal.addEventListener("abort", onAbort, { once: true }); let complete = false; try { if (context.signal.aborted) { @@ -255,7 +260,7 @@ export function createExecutorModelBroker(options: { } catch (error) { yield modelFailureOrThrow(error, context); } finally { - context.signal.removeEventListener("abort", cancel); + context.signal.removeEventListener("abort", onAbort); if (!complete) await cancel().catch(() => {}); reader.releaseLock(); } diff --git a/src/agent/hosted/executor-node-bootstrap.ts b/src/agent/hosted/executor-node-bootstrap.ts index c1e5680a6e..269bce3f2b 100644 --- a/src/agent/hosted/executor-node-bootstrap.ts +++ b/src/agent/hosted/executor-node-bootstrap.ts @@ -2,6 +2,7 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; import type { AddressInfo } from "node:net"; import process from "node:process"; +import { isNodeRuntime } from "#veryfront/platform/compat/runtime.ts"; import { tryResolve } from "#veryfront/extensions/contracts.ts"; import { createExecutorChannel, @@ -47,7 +48,8 @@ export interface ExecutorNodeBootstrap { const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; -function readBootstrap(environment: ExecutorBootstrapEnvironment) { +/** Validate the fixed Operator environment without enumerating or forwarding it. */ +export function readExecutorBootstrapConfiguration(environment: ExecutorBootstrapEnvironment) { try { const allocationId = environment.get("VERYFRONT_EXECUTOR_ALLOCATION_ID"); const generation = environment.get("VERYFRONT_EXECUTOR_GENERATION"); @@ -140,11 +142,11 @@ export async function startExecutorNodeBootstrap( options: ExecutorNodeBootstrapOptions, ): Promise { if ( - "Deno" in globalThis || "Bun" in globalThis || process.release.name !== "node" || + !isNodeRuntime() || process.release.name !== "node" || Number(process.versions.node.split(".")[0]) < 22 ) throw new Error("Executor bootstrap requires Node.js 22 or newer"); const startedAt = Date.now(); - const { binding, lifetimeMs, hardDeadlineAt } = readBootstrap( + const { binding, lifetimeMs, hardDeadlineAt } = readExecutorBootstrapConfiguration( options.environment ?? { get: (name) => process.env[name] }, ); const deadline = Math.min(startedAt + lifetimeMs, hardDeadlineAt); diff --git a/src/agent/hosted/executor-node-transport.ts b/src/agent/hosted/executor-node-transport.ts index 21d76d80d9..700a4d67e4 100644 --- a/src/agent/hosted/executor-node-transport.ts +++ b/src/agent/hosted/executor-node-transport.ts @@ -3,6 +3,7 @@ import { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; import { type AddressInfo, isIP, type Socket } from "node:net"; import process from "node:process"; +import { isNodeRuntime } from "#veryfront/platform/compat/runtime.ts"; import { connect, createServer, type TLSSocket } from "node:tls"; import type { ExecutorByteTransport } from "#veryfront/agent/executor/channel.ts"; import type { ExecutorBinding } from "#veryfront/agent/executor/protocol.ts"; @@ -58,7 +59,7 @@ function validateOptions( listen = false, ) { if ( - "Deno" in globalThis || "Bun" in globalThis || process.release.name !== "node" || + !isNodeRuntime() || process.release.name !== "node" || Number(process.versions.node.split(".")[0]) < 22 ) { throw new Error("Executor TLS transport requires Node.js 22 or newer"); diff --git a/src/agent/hosted/executor-persistence-bridge.test.ts b/src/agent/hosted/executor-persistence-bridge.test.ts new file mode 100644 index 0000000000..72a30184b0 --- /dev/null +++ b/src/agent/hosted/executor-persistence-bridge.test.ts @@ -0,0 +1,464 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { buildInvokeAgentChildRunLifecycleCustomEvent } from "#veryfront/agent/child-run/invoke-agent-child-runs.ts"; +import { createExecutorChannel, type ExecutorOperation } from "../executor/channel.ts"; +import { + createExecutorPersistenceBroker, + createExecutorPersistenceFacades, +} from "./executor-persistence-bridge.ts"; +import { + executorPersistenceJson, + executorPersistenceOperations, + getExecutorParentRunEventsRequestSchema, + getExecutorPersistenceCapabilityIdsSchema, +} from "./executor-persistence-schema.ts"; + +const binding = { + allocationId: "allocation-persistence-test", + generation: 7, + invocationId: "invocation-persistence-test", +}; +const capabilityIds = { + publishParentRunEvents: "parent-events-capability", + toolExposureCheckpoint: "tool-checkpoint-capability", + providerReplayCheckpoint: "provider-checkpoint-capability", +}; +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); +function parentProgressEvent( + status: "pending" | "running" | "waiting_for_tool" | "completed" | "failed" | "cancelled" = + "running", +) { + return buildInvokeAgentChildRunLifecycleCustomEvent({ + toolCallId: "tool-call-test", + childConversationId: "10000000-1000-4000-8000-100000000001", + childRunId: "child-run-test", + childMessageId: "10000000-1000-4000-8000-100000000002", + childAgentId: "child-agent-test", + status, + }); +} +const privateCheckpointEvents = [ + executorPersistenceJson({ + type: "AGENT_RUN_TOOL_EXPOSURE_CHECKPOINT", + version: 2, + loadedToolNames: ["search"], + }), + executorPersistenceJson({ + type: "AGENT_RUN_PROVIDER_REPLAY_CHECKPOINT", + version: 1, + messageId: "message-test", + provider: "anthropic", + providerBlocks: [], + providerBlockPositions: [], + totalPartCount: 1, + }), +]; + +function pair(operations: ReadonlyMap, maxConcurrentCalls?: number) { + const forward = new TransformStream(); + const backward = new TransformStream(); + const executor = createExecutorChannel({ + binding, + maxConcurrentCalls, + transport: { readable: backward.readable, writable: forward.writable }, + }); + const broker = createExecutorChannel({ + binding, + transport: { readable: forward.readable, writable: backward.writable }, + operations, + }); + return { + executor, + broker, + async close() { + executor.close(); + await broker.closed; + await Promise.all([executor.settled, broker.settled]); + }, + }; +} + +describe("executor persistence bridge", () => { + it("allows exact child progress events and rejects checkpoint event types", () => { + const request = { + capabilityId: capabilityIds.publishParentRunEvents, + sequence: 1, + events: [parentProgressEvent()], + }; + assertEquals(getExecutorParentRunEventsRequestSchema().safeParse(request).success, true); + for (const event of privateCheckpointEvents) { + assertEquals( + getExecutorParentRunEventsRequestSchema().safeParse({ ...request, events: [event] }) + .success, + false, + ); + } + }); + + it("rejects checkpoint event types before parent persistence dispatch", async () => { + let dispatches = 0; + const operations = createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + publishParentRunEvents: async () => { + dispatches++; + }, + }); + const operation = operations.get(executorPersistenceOperations.publishParentRunEvents); + if (operation?.mode !== "unary") throw new Error("missing synthetic operation"); + for (const event of privateCheckpointEvents) { + await assertRejects(() => + Promise.resolve(operation.handle({ + capabilityId: capabilityIds.publishParentRunEvents, + sequence: 1, + events: [event], + }, { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + })) + ); + } + assertEquals(dispatches, 0); + }); + + it("keeps later writes usable after an unsent request hits channel admission", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const written: string[] = []; + const channels = pair( + createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { publishParentRunEvents: "events" }, + publishParentRunEvents: async (events) => { + written.push(JSON.stringify(events[0])); + if (written.length === 1) { + entered.resolve(); + await release.promise; + } + }, + }), + 1, + ); + const facade = createExecutorPersistenceFacades({ + channel: channels.executor, + capabilityIds: { publishParentRunEvents: "events" }, + }); + try { + const firstEvent = parentProgressEvent("pending"); + const thirdEvent = parentProgressEvent("completed"); + const first = facade.publishParentRunEvents!([firstEvent]); + await entered.promise; + await assertRejects(() => facade.publishParentRunEvents!([parentProgressEvent("running")])); + release.resolve(); + await first; + await facade.publishParentRunEvents!([thirdEvent]); + assertEquals(written, [JSON.stringify(firstEvent), JSON.stringify(thirdEvent)]); + } finally { + release.resolve(); + await channels.close(); + } + }); + + it("persists canonical fractional checkpoint elapsed time", async () => { + let elapsed: number | undefined; + const channels = pair( + createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { providerReplayCheckpoint: "replay" }, + persistProviderReplayCheckpoint: async (checkpoint) => { + elapsed = checkpoint.elapsedMs; + }, + }), + ); + const facade = createExecutorPersistenceFacades({ + channel: channels.executor, + capabilityIds: { providerReplayCheckpoint: "replay" }, + }); + try { + await facade.providerReplayCheckpoint!.persist({ + version: 1, + messageId: "message-1", + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "synthetic" }, + }], + providerBlockPositions: [0], + totalPartCount: 1, + elapsedMs: 12.5, + }); + assertEquals(elapsed, 12.5); + } finally { + await channels.close(); + } + }); + it("acknowledges actual persistence in one invocation order", async () => { + const persisted: string[] = []; + const channels = pair(createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds, + publishParentRunEvents: async (events) => { + persisted.push(`events:${events[0]?.type}`); + }, + persistToolExposureCheckpoint: async (checkpoint) => { + persisted.push(`tools:${checkpoint.loadedToolNames.join(",")}`); + }, + persistProviderReplayCheckpoint: async (checkpoint) => { + persisted.push(`provider:${checkpoint.messageId}`); + }, + })); + try { + const facades = createExecutorPersistenceFacades({ + channel: channels.executor, + capabilityIds, + }); + await Promise.all([ + facades.publishParentRunEvents?.([parentProgressEvent()]), + facades.toolExposureCheckpoint?.persist({ version: 2, loadedToolNames: ["search"] }), + facades.providerReplayCheckpoint?.persist({ + version: 1, + messageId: "message-1", + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "synthetic" }, + }], + providerBlockPositions: [0], + providerMessageBlockCounts: [1], + totalPartCount: 1, + }), + ]); + assertEquals(persisted, ["events:CUSTOM", "tools:search", "provider:message-1"]); + } finally { + await channels.close(); + } + }); + + it("exports strict install capability IDs and never accepts authority fields", () => { + assertEquals(getExecutorPersistenceCapabilityIdsSchema().parse(capabilityIds), capabilityIds); + for (const field of ["runId", "ownerId", "authToken", "url"]) { + const result = getExecutorPersistenceCapabilityIdsSchema().safeParse({ + ...capabilityIds, + [field]: "synthetic-secret", + }); + assertEquals(result.success, false); + } + assertEquals( + getExecutorPersistenceCapabilityIdsSchema().safeParse({ + publishParentRunEvents: "duplicate-capability", + toolExposureCheckpoint: "duplicate-capability", + }).success, + false, + ); + }); + + it("does not consume sequence numbers for locally invalid or remotely unauthorized requests", async () => { + const persisted: string[] = []; + const operations = createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + publishParentRunEvents: async (events) => { + persisted.push(String(events[0]?.type)); + }, + }); + const operation = operations.get(executorPersistenceOperations.publishParentRunEvents); + if (operation?.mode !== "unary") throw new Error("missing synthetic operation"); + await assertRejects(() => + Promise.resolve(operation.handle({ + capabilityId: "wrong-capability", + sequence: 1, + events: [parentProgressEvent("pending")], + }, { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + })) + ); + + const channels = pair(operations); + try { + const facades = createExecutorPersistenceFacades({ + channel: channels.executor, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + }); + const cyclic: Record = { type: "INVALID" }; + cyclic.self = cyclic; + await assertRejects(() => + facades.publishParentRunEvents!( + [cyclic] as unknown as Parameters>[0], + ) + ); + await facades.publishParentRunEvents!([parentProgressEvent()]); + assertEquals(persisted, ["CUSTOM"]); + } finally { + await channels.close(); + } + }); + + it("withholds acknowledgement and channel settlement until the original write settles", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const channels = pair(createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + publishParentRunEvents: async () => { + entered.resolve(); + await release.promise; + }, + })); + const facades = createExecutorPersistenceFacades({ + channel: channels.executor, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + }); + let acknowledged = false; + const request = facades.publishParentRunEvents!([parentProgressEvent("completed")]).then(() => { + acknowledged = true; + }); + await entered.promise; + await tick(); + assertEquals(acknowledged, false); + + channels.executor.close(); + await channels.broker.closed; + let settled = false; + void channels.broker.settled.then(() => settled = true); + await assertRejects(() => request); + await tick(); + assertEquals(settled, false); + release.resolve(); + await Promise.all([channels.executor.settled, channels.broker.settled]); + assertEquals(settled, true); + }); + + it("fails closed on incomplete capabilities and validates initial checkpoint copies", () => { + assertThrows(() => + createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { toolExposureCheckpoint: capabilityIds.toolExposureCheckpoint }, + }) + ); + assertThrows(() => + createExecutorPersistenceFacades({ + channel: {} as Parameters[0]["channel"], + capabilityIds: {}, + initialToolExposureCheckpoint: { version: 2, loadedToolNames: ["search"] }, + }) + ); + }); + + it("consumes accepted failed writes once and never retries them", async () => { + const persisted: string[] = []; + const operations = createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { publishParentRunEvents: capabilityIds.publishParentRunEvents }, + publishParentRunEvents: async (events) => { + persisted.push(String(events[0]?.type)); + if (persisted.length === 1) throw new Error("synthetic persistence failure"); + }, + }); + const operation = operations.get(executorPersistenceOperations.publishParentRunEvents); + if (operation?.mode !== "unary") throw new Error("missing synthetic operation"); + const context = { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }; + await assertRejects(() => + Promise.resolve(operation.handle({ + capabilityId: capabilityIds.publishParentRunEvents, + sequence: 1, + events: [parentProgressEvent("pending")], + }, context)) + ); + await assertRejects(() => + Promise.resolve(operation.handle({ + capabilityId: capabilityIds.publishParentRunEvents, + sequence: 1, + events: [parentProgressEvent("running")], + }, context)) + ); + assertEquals( + await operation.handle({ + capabilityId: capabilityIds.publishParentRunEvents, + sequence: 2, + events: [parentProgressEvent("completed")], + }, context), + { acknowledged: true, sequence: 2 }, + ); + assertEquals(persisted, ["CUSTOM", "CUSTOM"]); + }); + + it("rejects malformed replay state and mismatched bindings without reserving sequence", async () => { + let writes = 0; + const operations = createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: { providerReplayCheckpoint: capabilityIds.providerReplayCheckpoint }, + persistProviderReplayCheckpoint: () => { + writes++; + }, + }); + const operation = operations.get(executorPersistenceOperations.persistProviderReplayCheckpoint); + if (operation?.mode !== "unary") throw new Error("missing synthetic operation"); + const checkpoint = { + version: 1, + messageId: "message-1", + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "synthetic" }, + }], + providerBlockPositions: [0], + providerMessageBlockCounts: [1], + totalPartCount: 1, + } as const; + await assertRejects(() => + Promise.resolve(operation.handle( + executorPersistenceJson({ + capabilityId: capabilityIds.providerReplayCheckpoint, + sequence: 1, + checkpoint: { ...checkpoint, providerBlockPositions: [1] }, + }), + { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }, + )) + ); + await assertRejects(() => + Promise.resolve(operation.handle( + executorPersistenceJson({ + capabilityId: capabilityIds.providerReplayCheckpoint, + sequence: 1, + checkpoint, + }), + { + binding: { ...binding, invocationId: "other-invocation" }, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }, + )) + ); + assertEquals( + await operation.handle( + executorPersistenceJson({ + capabilityId: capabilityIds.providerReplayCheckpoint, + sequence: 1, + checkpoint, + }), + { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }, + ), + { acknowledged: true, sequence: 1 }, + ); + assertEquals(writes, 1); + }); +}); diff --git a/src/agent/hosted/executor-persistence-bridge.ts b/src/agent/hosted/executor-persistence-bridge.ts new file mode 100644 index 0000000000..49bc333613 --- /dev/null +++ b/src/agent/hosted/executor-persistence-bridge.ts @@ -0,0 +1,285 @@ +import type { ConversationRunEvent } from "#veryfront/agent/conversation/run-events.ts"; +import type { Schema } from "#veryfront/extensions/schema/index.ts"; +import type { ProviderReplayCheckpoint } from "#veryfront/agent/runtime/provider-replay.ts"; +import { + copyExecutorReplayCheckpoints, + createExecutorCheckpointStateOperations, + type ExecutorInitialCheckpointState, +} from "./executor-checkpoint-state.ts"; +import { parseProviderReplayCheckpoint } from "#veryfront/agent/runtime/provider-replay.ts"; +import type { ToolExposureCheckpoint } from "#veryfront/agent/runtime/tool-exposure.ts"; +import type { ExecutorBinding } from "../executor/protocol.ts"; +import { getExecutorBindingSchema } from "../executor/protocol.ts"; +import type { + ExecutorChannel, + ExecutorOperation, + ExecutorOperationContext, +} from "../executor/channel.ts"; +import type { ExecutorRuntimeFacades } from "./executor-runtime-prepare.ts"; +import { + type ExecutorPersistenceCapabilityIds, + executorPersistenceJson, + executorPersistenceOperations, + getExecutorParentRunEventsRequestSchema, + getExecutorPersistenceAckSchema, + getExecutorPersistenceCapabilityIdsSchema, + getExecutorProviderReplayCheckpointRequestSchema, + getExecutorToolExposureCheckpointRequestSchema, + getExecutorToolExposureCheckpointSchema, + parseExecutorPersistenceData, +} from "./executor-persistence-schema.ts"; +export type { ExecutorPersistenceCapabilityIds } from "./executor-persistence-schema.ts"; + +/** + * Non-owning views over an authenticated channel. Each promise resolves only + * after a broker acknowledgement. The session owner closes the shared channel + * and retains `channel.settled`; these facades own no separate cleanup task. + */ +export type ExecutorPersistenceFacades = Pick< + ExecutorRuntimeFacades, + "publishParentRunEvents" | "toolExposureCheckpoint" | "providerReplayCheckpoint" +>; + +type ExecutorPersistencePayload = + | { events: ConversationRunEvent[] } + | { checkpoint: ToolExposureCheckpoint | ProviderReplayCheckpoint }; + +function sameBinding(left: Readonly, right: Readonly): boolean { + return left.allocationId === right.allocationId && left.generation === right.generation && + left.invocationId === right.invocationId; +} + +function snapshotCapabilityIds( + value: ExecutorPersistenceCapabilityIds, +): ExecutorPersistenceCapabilityIds { + const result = parseExecutorPersistenceData(getExecutorPersistenceCapabilityIdsSchema(), value); + return Object.freeze(result); +} + +async function awaitPersistence( + persistence: Promise, + signal: AbortSignal, +): Promise { + const aborted = Promise.withResolvers(); + const onAbort = () => aborted.reject(new TypeError("Managed persistence cancelled")); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + try { + await Promise.race([persistence, aborted.promise]); + signal.throwIfAborted(); + } finally { + signal.removeEventListener("abort", onAbort); + // Cancellation releases the caller independently. The handler retains the + // original write so channel settlement still represents durable settlement. + await persistence.catch(() => {}); + } +} + +/** + * Install trusted persistence handlers. Run identity and credentials remain in + * the supplied closures and are absent from every strict channel DTO. + */ +export function createExecutorPersistenceBroker( + options: ExecutorInitialCheckpointState & { + expectedBinding: ExecutorBinding; + capabilityIds: ExecutorPersistenceCapabilityIds; + publishParentRunEvents?: NonNullable; + persistToolExposureCheckpoint?: NonNullable< + NonNullable["persist"] + >; + persistProviderReplayCheckpoint?: NonNullable< + NonNullable["persist"] + >; + }, +): ReadonlyMap { + const expectedBinding = Object.freeze(getExecutorBindingSchema().parse(options.expectedBinding)); + const capabilityIds = snapshotCapabilityIds(options.capabilityIds); + const definitions = [ + ["publishParentRunEvents", options.publishParentRunEvents], + ["toolExposureCheckpoint", options.persistToolExposureCheckpoint], + ["providerReplayCheckpoint", options.persistProviderReplayCheckpoint], + ] as const; + for (const [name, callback] of definitions) { + if ((capabilityIds[name] === undefined) !== (callback === undefined)) { + throw new TypeError("Managed persistence capability configuration is incomplete"); + } + } + + let lastAcceptedSequence = 0; + let persistenceTail = Promise.resolve(); + const authorize = ( + capabilityId: string, + sequence: number, + expectedCapabilityId: string, + context: ExecutorOperationContext, + ) => { + if ( + !sameBinding(expectedBinding, context.binding) || capabilityId !== expectedCapabilityId || + sequence <= lastAcceptedSequence + ) throw new TypeError("Managed persistence operation is not authorized"); + // Unsent calls can consume a client sequence before channel admission. + // Gaps are safe; replays and out-of-order accepted writes are forbidden. + lastAcceptedSequence = sequence; + }; + const persist = async ( + sequence: number, + context: ExecutorOperationContext, + write: () => void | Promise, + ) => { + const persistence = persistenceTail.then(write); + persistenceTail = persistence.catch(() => {}); + await awaitPersistence(persistence, context.signal); + return executorPersistenceJson({ acknowledged: true, sequence }); + }; + const operations = new Map( + createExecutorCheckpointStateOperations({ ...options, expectedBinding, capabilityIds }), + ); + if (capabilityIds.publishParentRunEvents && options.publishParentRunEvents) { + const capabilityId = capabilityIds.publishParentRunEvents; + const publish = options.publishParentRunEvents; + operations.set(executorPersistenceOperations.publishParentRunEvents, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorPersistenceData( + getExecutorParentRunEventsRequestSchema(), + value, + ); + authorize(request.capabilityId, request.sequence, capabilityId, context); + return await persist(request.sequence, context, () => publish(request.events)); + }, + }); + } + if (capabilityIds.toolExposureCheckpoint && options.persistToolExposureCheckpoint) { + const capabilityId = capabilityIds.toolExposureCheckpoint; + const persistCheckpoint = options.persistToolExposureCheckpoint; + operations.set(executorPersistenceOperations.persistToolExposureCheckpoint, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorPersistenceData( + getExecutorToolExposureCheckpointRequestSchema(), + value, + ); + authorize(request.capabilityId, request.sequence, capabilityId, context); + return await persist( + request.sequence, + context, + () => persistCheckpoint(request.checkpoint), + ); + }, + }); + } + if (capabilityIds.providerReplayCheckpoint && options.persistProviderReplayCheckpoint) { + const capabilityId = capabilityIds.providerReplayCheckpoint; + const persistCheckpoint = options.persistProviderReplayCheckpoint; + operations.set(executorPersistenceOperations.persistProviderReplayCheckpoint, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorPersistenceData( + getExecutorProviderReplayCheckpointRequestSchema(), + value, + ); + const checkpoint = parseProviderReplayCheckpoint(request.checkpoint); + authorize(request.capabilityId, request.sequence, capabilityId, context); + return await persist(request.sequence, context, () => persistCheckpoint(checkpoint)); + }, + }); + } + return operations; +} + +/** Create executor-local facades over one authenticated invocation channel. */ +export function createExecutorPersistenceFacades(options: { + channel: ExecutorChannel; + capabilityIds: ExecutorPersistenceCapabilityIds; + /** Optional view lifetime; does not transfer ownership of the shared channel. */ + signal?: AbortSignal; + initialToolExposureCheckpoint?: ToolExposureCheckpoint; + initialProviderReplayCheckpoints?: readonly ProviderReplayCheckpoint[]; +}): ExecutorPersistenceFacades { + const capabilityIds = snapshotCapabilityIds(options.capabilityIds); + if (options.initialToolExposureCheckpoint && !capabilityIds.toolExposureCheckpoint) { + throw new TypeError("Managed tool checkpoint capability is required"); + } + if (options.initialProviderReplayCheckpoints && !capabilityIds.providerReplayCheckpoint) { + throw new TypeError("Managed provider checkpoint capability is required"); + } + const initialToolExposureCheckpoint = options.initialToolExposureCheckpoint === undefined + ? undefined + : parseExecutorPersistenceData( + getExecutorToolExposureCheckpointSchema(), + executorPersistenceJson(options.initialToolExposureCheckpoint), + ); + const initialProviderReplayCheckpoints = options.initialProviderReplayCheckpoints === undefined + ? undefined + : copyExecutorReplayCheckpoints(options.initialProviderReplayCheckpoints); + let sequence = 0; + const request = async ( + operation: string, + capabilityId: string, + payload: ExecutorPersistencePayload, + schema: Schema, + ) => { + options.signal?.throwIfAborted(); + if (sequence === Number.MAX_SAFE_INTEGER) { + throw new TypeError("Managed persistence call limit exceeded"); + } + const callSequence = sequence + 1; + const input = executorPersistenceJson({ capabilityId, sequence: callSequence, ...payload }); + parseExecutorPersistenceData(schema, input); + sequence = callSequence; + const result = await options.channel.request( + operation, + input, + { signal: options.signal }, + ); + const ack = parseExecutorPersistenceData(getExecutorPersistenceAckSchema(), result); + if (ack.sequence !== callSequence) { + throw new TypeError("Invalid managed persistence acknowledgement"); + } + }; + return { + ...(capabilityIds.publishParentRunEvents + ? { + publishParentRunEvents: (events: ConversationRunEvent[]) => + request( + executorPersistenceOperations.publishParentRunEvents, + capabilityIds.publishParentRunEvents!, + { events }, + getExecutorParentRunEventsRequestSchema(), + ), + } + : {}), + ...(capabilityIds.toolExposureCheckpoint + ? { + toolExposureCheckpoint: { + initial: initialToolExposureCheckpoint, + persist: (checkpoint: ToolExposureCheckpoint) => + request( + executorPersistenceOperations.persistToolExposureCheckpoint, + capabilityIds.toolExposureCheckpoint!, + { checkpoint }, + getExecutorToolExposureCheckpointRequestSchema(), + ), + }, + } + : {}), + ...(capabilityIds.providerReplayCheckpoint + ? { + providerReplayCheckpoint: { + initial: initialProviderReplayCheckpoints, + persist: (checkpoint: ProviderReplayCheckpoint) => { + const parsed = parseProviderReplayCheckpoint( + executorPersistenceJson(checkpoint), + ); + return request( + executorPersistenceOperations.persistProviderReplayCheckpoint, + capabilityIds.providerReplayCheckpoint!, + { checkpoint: parsed }, + getExecutorProviderReplayCheckpointRequestSchema(), + ); + }, + }, + } + : {}), + }; +} diff --git a/src/agent/hosted/executor-persistence-schema.ts b/src/agent/hosted/executor-persistence-schema.ts new file mode 100644 index 0000000000..a68b6ab898 --- /dev/null +++ b/src/agent/hosted/executor-persistence-schema.ts @@ -0,0 +1,141 @@ +import type { InferSchema, Schema } from "#veryfront/extensions/schema/index.ts"; +import { defineSchema, getJsonValueSchema, type JsonValue } from "#veryfront/schemas/index.ts"; +import { snapshotBoundedJsonValue } from "#veryfront/schemas/json-value.ts"; +import { MAX_CONVERSATION_RUN_EVENT_PAYLOAD_BYTES } from "#veryfront/agent/conversation/run-event-limits.ts"; +import { + getInvokeAgentChildRunLifecycleCustomEventSchema, + getInvokeAgentChildRunStateDeltaSchema, +} from "#veryfront/agent/child-run/invoke-agent-child-runs.ts"; +import { getExecutorDiscoveryIdSchema } from "./executor-discovery-schema.ts"; +import { EXECUTOR_MAX_FRAME_BYTES } from "../executor/protocol.ts"; + +const MAX_PERSISTENCE_ITEMS = 1_000; +const MAX_TOOL_NAMES = 4_096; +const MAX_PROVIDER_BLOCKS = 100; +const MAX_PROVIDER_PARTS = 10_000; +const MAX_PERSISTENCE_PAYLOAD_BYTES = EXECUTOR_MAX_FRAME_BYTES - 2_048; +const encoder = new TextEncoder(); + +export const executorPersistenceOperations = Object.freeze( + { + publishParentRunEvents: "persistence.parent-run-events", + persistToolExposureCheckpoint: "persistence.tool-exposure-checkpoint", + persistProviderReplayCheckpoint: "persistence.provider-replay-checkpoint", + } as const, +); + +const getSequenceSchema = defineSchema((v) => + v.number().int().positive().max(Number.MAX_SAFE_INTEGER) +); +const getCapabilityRequestSchema = defineSchema((v) => + v.object({ + capabilityId: getExecutorDiscoveryIdSchema(), + sequence: getSequenceSchema(), + }).strict() +); +const getExecutorParentRunEventSchema = defineSchema((v) => + v.union([ + getInvokeAgentChildRunStateDeltaSchema(), + getInvokeAgentChildRunLifecycleCustomEventSchema(), + ]) +); + +/** Identifiers installed by the broker; authority and run ownership are never wire fields. */ +export const getExecutorPersistenceCapabilityIdsSchema = defineSchema((v) => + v.object({ + publishParentRunEvents: getExecutorDiscoveryIdSchema().optional(), + toolExposureCheckpoint: getExecutorDiscoveryIdSchema().optional(), + providerReplayCheckpoint: getExecutorDiscoveryIdSchema().optional(), + }).strict().refine((value) => { + const ids = Object.values(value).filter((id): id is string => id !== undefined); + return new Set(ids).size === ids.length; + }, "Managed persistence capability IDs must be distinct") +); +export type ExecutorPersistenceCapabilityIds = InferSchema< + ReturnType +>; + +export const getExecutorParentRunEventsRequestSchema = defineSchema((v) => + getCapabilityRequestSchema().extend({ + events: v.array( + getExecutorParentRunEventSchema().refine((event) => + encoder.encode(JSON.stringify(event)).byteLength <= + MAX_CONVERSATION_RUN_EVENT_PAYLOAD_BYTES + ), + ).max(MAX_PERSISTENCE_ITEMS), + }).strict() +); + +export const getExecutorToolExposureCheckpointSchema = defineSchema((v) => + v.object({ + version: v.union([v.literal(1), v.literal(2)]), + loadedToolNames: v.array(v.string().min(1).max(256)).max(MAX_TOOL_NAMES), + }).strict() +); + +export const getExecutorToolExposureCheckpointRequestSchema = defineSchema((_v) => + getCapabilityRequestSchema().extend({ + checkpoint: getExecutorToolExposureCheckpointSchema(), + }).strict() +); + +export const getExecutorProviderReplayCheckpointSchema = defineSchema((v) => { + const provider = v.enum(["anthropic", "openai-responses"] as const); + return v.object({ + version: v.literal(1), + messageId: v.string().min(1).max(256), + provider, + providerBlocks: v.array( + v.object({ + type: v.literal("provider-block"), + provider, + block: v.record(v.string(), getJsonValueSchema()), + }).strict(), + ).min(1).max(MAX_PROVIDER_BLOCKS), + providerBlockPositions: v.array(v.number().int().nonnegative().max(MAX_PROVIDER_PARTS - 1)) + .min(1).max(MAX_PROVIDER_BLOCKS), + providerMessageBlockCounts: v.array(v.number().int().positive().max(MAX_PROVIDER_BLOCKS)) + .min(1).max(MAX_PROVIDER_BLOCKS).optional(), + totalPartCount: v.number().int().positive().max(MAX_PROVIDER_PARTS), + elapsedMs: v.number().nonnegative().refine(Number.isFinite).optional(), + emittedAt: v.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), + }).strict(); +}); + +export const getExecutorProviderReplayCheckpointRequestSchema = defineSchema((_v) => + getCapabilityRequestSchema().extend({ + checkpoint: getExecutorProviderReplayCheckpointSchema(), + }).strict() +); + +export const getExecutorPersistenceAckSchema = defineSchema((v) => + v.object({ acknowledged: v.literal(true), sequence: getSequenceSchema() }).strict() +); + +export type ExecutorPersistenceAck = InferSchema< + ReturnType +>; + +/** Fixed diagnostics do not echo rejected events or opaque replay metadata. */ +export function parseExecutorPersistenceData(schema: Schema, value: unknown): T { + const result = schema.safeParse(value); + if (!result.success) throw new TypeError("Invalid managed persistence data"); + return result.data; +} + +/** Copy a validated persistence payload while enforcing the channel envelope budget. */ +export function executorPersistenceJson(value: unknown): JsonValue { + let encoded: string | undefined; + try { + encoded = JSON.stringify(value); + } catch { + throw new TypeError("Invalid managed persistence data"); + } + if ( + encoded === undefined || + encoder.encode(encoded).byteLength > MAX_PERSISTENCE_PAYLOAD_BYTES + ) throw new TypeError("Invalid managed persistence data"); + const snapshot = snapshotBoundedJsonValue(JSON.parse(encoded)); + if (!snapshot.success) throw new TypeError("Invalid managed persistence data"); + return snapshot.value; +} diff --git a/src/agent/hosted/executor-runtime-contracts.ts b/src/agent/hosted/executor-runtime-contracts.ts new file mode 100644 index 0000000000..ab34e2e9d6 --- /dev/null +++ b/src/agent/hosted/executor-runtime-contracts.ts @@ -0,0 +1,55 @@ +import { ensureBuiltinSchemaValidator } from "#veryfront/extensions/builtin-schema-validator.ts"; +import { ensureDefaultBundlerContracts } from "#veryfront/extensions/bundler/defaults.ts"; +import type { Bundler } from "#veryfront/extensions/bundler/bundler.ts"; +import type { ModuleLexer } from "#veryfront/extensions/bundler/module-lexer.ts"; +import { tryResolve } from "#veryfront/extensions/contracts.ts"; +import { + ensureDefaultSkillDocumentParserContract, +} from "#veryfront/extensions/parser/skill-defaults.ts"; +import { + type SkillDocumentParserProvider, + SkillDocumentParserProviderName, +} from "#veryfront/extensions/parser/skill-document-parser.ts"; +import type { SchemaValidator } from "#veryfront/extensions/schema/index.ts"; + +let initialization: Promise | undefined; + +function assertRuntimeContracts(): void { + const schema = tryResolve("SchemaValidator"); + const bundler = tryResolve("Bundler"); + const lexer = tryResolve("ModuleLexer"); + const skillParser = tryResolve(SkillDocumentParserProviderName); + if ( + !schema || typeof schema.string !== "function" || typeof schema.object !== "function" || + !bundler || typeof bundler.bundle !== "function" || typeof bundler.transform !== "function" || + !lexer || typeof lexer.parse !== "function" || + !skillParser || typeof skillParser.parseFrontmatter !== "function" + ) throw new TypeError("Executor runtime contracts are unavailable"); +} + +/** + * Install the fixed first-party contracts required before executor project imports. + * Concurrent and repeated startup preserves any already-registered trusted generation. + */ +export async function initializeExecutorRuntimeContracts(): Promise { + try { + assertRuntimeContracts(); + return; + } catch { + // Missing contracts are initialized below; malformed registered contracts + // remain authoritative and fail the final validation instead of being replaced. + } + initialization ??= (async () => { + ensureBuiltinSchemaValidator(); + await Promise.all([ + ensureDefaultSkillDocumentParserContract(), + // Uses the fixed ext-bundler-esbuild first-party import and rechecks the + // registry after asynchronous module loading before registering. + ensureDefaultBundlerContracts(), + ]); + assertRuntimeContracts(); + })().finally(() => { + initialization = undefined; + }); + await initialization; +} diff --git a/src/agent/hosted/executor-runtime-entrypoint.ts b/src/agent/hosted/executor-runtime-entrypoint.ts new file mode 100644 index 0000000000..de4fdbd935 --- /dev/null +++ b/src/agent/hosted/executor-runtime-entrypoint.ts @@ -0,0 +1,157 @@ +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; +import { isAbsolute } from "node:path"; +import process from "node:process"; +import { tryResolve } from "#veryfront/extensions/contracts.ts"; +import type { ExecutorChannel } from "../executor/channel.ts"; +import { + type ExecutorNodeBootstrapOptions, + readExecutorBootstrapConfiguration, + startExecutorNodeBootstrap, +} from "./executor-node-bootstrap.ts"; +import { createExecutorRuntimeInstallation } from "./executor-runtime-install.ts"; +import { awaitExecutorCleanup } from "./executor-runtime-settlement.ts"; +export { initializeExecutorRuntimeContracts } from "./executor-runtime-contracts.ts"; +import { + type ExecutorArtifactManifest, + getExecutorArtifactManifestSchema, + parseExecutorInstallation, +} from "./executor-runtime-install-schema.ts"; + +const ARTIFACT_MANIFEST = "/opt/veryfront/executor-artifact.json"; +const PROJECT_ROOT = "/opt/veryfront/project"; + +async function readFixedArtifact() { + const file = await open( + ARTIFACT_MANIFEST, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.uid !== 0 || (stat.mode & 0o022) !== 0 || stat.size > 64 * 1024) { + throw new Error("Invalid executor artifact"); + } + const buffer = new Uint8Array(64 * 1024 + 1); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await file.read(buffer, offset, buffer.length - offset, offset); + if (!bytesRead) break; + offset += bytesRead; + } + if (offset > 64 * 1024) throw new Error("Invalid executor artifact"); + const manifest = parseExecutorInstallation( + getExecutorArtifactManifestSchema(), + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset))), + ); + return { manifest, projectDir: PROJECT_ROOT }; + } catch { + throw new Error("Invalid executor artifact"); + } finally { + await file.close(); + } +} + +/** + * Dedicated executor entrypoint. The reviewed image launcher registers its + * first-party SchemaValidator, Bundler, ModuleLexer and SkillDocumentParserProvider before calling + * this function. The fixed image + * manifest is outside the project tree and is never selected by channel input. + */ +export async function startExecutorRuntimeEntrypoint( + options: Pick & { + /** Trusted image/test boundary, never a channel field or environment path. */ + readArtifact?: () => Promise<{ manifest: ExecutorArtifactManifest; projectDir: string }>; + } = {}, +) { + const environment = options.environment ?? { get: (name) => process.env[name] }; + const { binding } = readExecutorBootstrapConfiguration(environment); + for ( + const contract of ["SchemaValidator", "Bundler", "ModuleLexer", "SkillDocumentParserProvider"] + ) { + if (!tryResolve(contract)) throw new TypeError("Executor runtime contracts are unavailable"); + } + const artifact = await (options.readArtifact ?? readFixedArtifact)(); + if (!isAbsolute(artifact.projectDir)) throw new TypeError("Invalid executor project root"); + const lifetime = new AbortController(); + const signal = AbortSignal.any([lifetime.signal, ...(options.signal ? [options.signal] : [])]); + signal.throwIfAborted(); + const channel = Promise.withResolvers(); + void channel.promise.catch(() => {}); + const installation = createExecutorRuntimeInstallation({ + binding, + artifact: artifact.manifest, + signal, + async install(input, runtimeSignal) { + // No project discovery, runtime factories or capability construction is + // evaluated until the authenticated one-shot installation has passed. + const { createExecutorRuntimeFacades } = await import("./executor-runtime-facades.ts"); + const facades = await createExecutorRuntimeFacades({ + input, + channel: await channel.promise, + signal: runtimeSignal, + }); + let discovery: import("./executor-discovery.ts").ExecutorDiscovery | undefined; + try { + runtimeSignal.throwIfAborted(); + const { createExecutorDiscovery } = await import("./executor-discovery.ts"); + discovery = createExecutorDiscovery({ + binding, + source: input.source, + projectDir: artifact.projectDir, + defaultAgentId: input.grant.agentId, + signal: runtimeSignal, + }); + const { createExecutorRuntimePreparation } = await import("./executor-runtime-prepare.ts"); + runtimeSignal.throwIfAborted(); + return createExecutorRuntimePreparation({ + binding, + source: input.source, + discovery, + facades, + grant: { + ...input.grant, + models: new Map(input.grant.models.map(({ id, ...policy }) => [id, policy])), + }, + }); + } catch (error) { + await Promise.allSettled([facades.cleanup(), discovery?.close()]); + throw error; + } + }, + }); + try { + const bootstrap = await startExecutorNodeBootstrap({ + ...options, + environment, + operations: installation.operations, + signal, + }); + void bootstrap.ready.then(channel.resolve, channel.reject); + void bootstrap.ready.then(async (connected) => { + await connected.closed; + lifetime.abort(); + }).catch(() => lifetime.abort()); + let closing: Promise | undefined; + const close = (): Promise => { + if (!closing) { + closing = awaitExecutorCleanup([ + Promise.resolve().then(() => bootstrap.close()), + Promise.resolve().then(() => installation.close()), + channel.promise.then((connected) => connected.settled, () => {}), + ]); + lifetime.abort(); + } + return closing; + }; + const settled = awaitExecutorCleanup([ + installation.settled, + channel.promise.then((connected) => connected.settled, () => {}), + ]); + void settled.catch(() => {}); + return { address: bootstrap.address, ready: bootstrap.ready, close, settled }; + } catch (error) { + channel.reject(new Error("Executor startup failed")); + await installation.close(); + throw error; + } +} diff --git a/src/agent/hosted/executor-runtime-facades.test.ts b/src/agent/hosted/executor-runtime-facades.test.ts new file mode 100644 index 0000000000..e654ad7d45 --- /dev/null +++ b/src/agent/hosted/executor-runtime-facades.test.ts @@ -0,0 +1,232 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; +import type { ExecutorOperation } from "../executor/channel.ts"; +import type { JsonValue } from "#veryfront/schemas/index.ts"; +import { createExecutorChannel } from "../executor/channel.ts"; +import type { ExecutorRuntimeInstall } from "./executor-runtime-install-schema.ts"; +import { createExecutorRuntimeFacades } from "./executor-runtime-facades.ts"; +import { createExecutorPersistenceBroker } from "./executor-persistence-bridge.ts"; +import type { ProviderReplayCheckpoint } from "../runtime/provider-replay.ts"; + +const binding = { + allocationId: "facade-allocation", + invocationId: "facade-invocation", + generation: 1, +}; +const modelId = "veryfront-cloud/openai/synthetic-model"; +function installation(): ExecutorRuntimeInstall { + return { + version: 1, + binding, + root: "project", + owner: { scopeKind: "global", serviceName: "veryfront-agent" }, + source: { type: "release", releaseId: "release-1" }, + grant: { + agentId: "coder", + defaultModelId: modelId, + maxSteps: 3, + models: [{ id: modelId, maxOutputTokens: 100, providerToolNames: [] }], + allowedToolNames: ["read"], + hostToolFacadeIds: ["host"], + remoteToolSourceIds: ["remote"], + execution: { kind: "ephemeral", projectId: null }, + }, + capabilities: { persistence: {} }, + }; +} +function channels( + sources = ["host", "remote"], + persistenceOperations: ReadonlyMap = new Map(), +) { + const toBroker = new TransformStream(); + const toExecutor = new TransformStream(); + let executions = 0; + let writes = 0; + const operations = new Map([ + ["persistence.initial-checkpoints", { + mode: "stream", + async *handle() { + yield { type: "complete" }; + }, + }], + ["persistence.tool-exposure-checkpoint", { + mode: "unary", + handle(input) { + writes++; + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new Error("Invalid fixture request"); + } + return { acknowledged: true, sequence: input.sequence! }; + }, + }], + ["model.metadata", { + mode: "unary", + handle: () => [{ + id: modelId, + modelId: "synthetic-model", + provider: "openai", + specificationVersion: "v3", + }], + }], + ["tool.sources", { + mode: "stream", + async *handle(): AsyncGenerator { + for (const sourceId of sources) yield { type: "source", sourceId }; + yield { type: "complete" }; + }, + }], + ["tool.list", { + mode: "stream", + async *handle(): AsyncGenerator { + yield { + type: "tool", + definition: { + name: "read", + description: "Read a synthetic value", + parameters: { type: "object", properties: {} }, + }, + }; + yield { type: "complete" }; + }, + }], + ["tool.execute", { + mode: "stream", + async *handle() { + executions++; + yield { type: "result", result: "synthetic-result" }; + }, + }], + ]); + for (const [name, operation] of persistenceOperations) operations.set(name, operation); + const broker = createExecutorChannel({ + binding, + operations, + transport: { readable: toBroker.readable, writable: toExecutor.writable }, + }); + const executor = createExecutorChannel({ + binding, + transport: { readable: toExecutor.readable, writable: toBroker.writable }, + }); + return { + broker, + executor, + get executions() { + return executions; + }, + get writes() { + return writes; + }, + async close() { + broker.close(); + executor.close(); + await Promise.all([broker.settled, executor.settled]); + }, + }; +} + +it("constructs model and host/remote tool facades only for the installed IDs", async () => { + const pair = channels(); + try { + const facades = await createExecutorRuntimeFacades({ + input: installation(), + channel: pair.executor, + signal: pair.executor.signal, + }); + assertEquals([...facades.hostTools.keys()], ["host"]); + assertEquals([...facades.remoteToolSources.keys()], ["remote"]); + const read = facades.hostTools.get("host")!.read!; + assertEquals(read.inputSchemaJson, { type: "object", properties: {} }); + assertEquals(await read.execute!({}, { toolCallId: "call-1" }), "synthetic-result"); + assertEquals(pair.executions, 1); + await facades.cleanup(); + await assertRejects(async () => { + await read.execute!({}, { toolCallId: "late" }); + }); + assertEquals(pair.executions, 1); + } finally { + await pair.close(); + } +}); + +it("rejects an incomplete or expanded tool source grant", async () => { + for (const sources of [["host"], ["host", "remote", "ungranted"]]) { + const pair = channels(sources); + try { + await assertRejects(() => + createExecutorRuntimeFacades({ + input: installation(), + channel: pair.executor, + signal: pair.executor.signal, + }) + ); + assertEquals(pair.executions, 0); + } finally { + await pair.close(); + } + } +}); + +it("revokes persistence facades on runtime cleanup without closing the shared channel", async () => { + const pair = channels(); + const input = installation(); + input.capabilities.persistence.toolExposureCheckpoint = "checkpoint"; + try { + const facades = await createExecutorRuntimeFacades({ + input, + channel: pair.executor, + signal: pair.executor.signal, + }); + const persist = facades.toolExposureCheckpoint!.persist; + await persist({ version: 1, loadedToolNames: [] }); + await facades.cleanup(); + await assertRejects(async () => { + await persist({ version: 1, loadedToolNames: [] }); + }); + assertEquals(pair.writes, 1); + assertEquals(pair.executor.signal.aborted, false); + } finally { + await pair.close(); + } +}); + +it("restores a durable replay checkpoint larger than the installation envelope", async () => { + const checkpoint: ProviderReplayCheckpoint = { + version: 1, + messageId: "message-large", + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "x".repeat(70_000) }, + }], + providerBlockPositions: [0], + totalPartCount: 1, + elapsedMs: 12.5, + }; + const input = installation(); + input.capabilities.persistence.providerReplayCheckpoint = "replay"; + const pair = channels( + ["host", "remote"], + createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: input.capabilities.persistence, + persistProviderReplayCheckpoint: () => Promise.resolve(), + initialProviderReplayCheckpoints: [checkpoint], + }), + ); + try { + const facades = await createExecutorRuntimeFacades({ + input, + channel: pair.executor, + signal: pair.executor.signal, + }); + assertEquals(facades.providerReplayCheckpoint?.initial?.[0]?.messageId, "message-large"); + const data = facades.providerReplayCheckpoint?.initial?.[0]?.providerBlocks[0]?.block.data; + assert(typeof data === "string"); + assertEquals(data.length, 70_000); + await facades.cleanup(); + } finally { + await pair.close(); + } +}); diff --git a/src/agent/hosted/executor-runtime-facades.ts b/src/agent/hosted/executor-runtime-facades.ts new file mode 100644 index 0000000000..41b48f52c5 --- /dev/null +++ b/src/agent/hosted/executor-runtime-facades.ts @@ -0,0 +1,106 @@ +import type { ExecutorChannel } from "../executor/channel.ts"; +import type { HostToolSet } from "#veryfront/tool/host-tools.ts"; +import type { RemoteToolSource, ToolExecutionContext } from "#veryfront/tool/types.ts"; +import { revokeModelRuntimeResolver } from "../runtime/model-transport.ts"; +import type { ExecutorRuntimeFacades } from "./executor-runtime-prepare.ts"; +import type { ExecutorRuntimeInstall } from "./executor-runtime-install-schema.ts"; +import { createExecutorModelRuntimeResolver } from "./executor-model-bridge.ts"; +import { createExecutorRemoteToolSources } from "./executor-tool-remote-facade.ts"; +import { createExecutorPersistenceFacades } from "./executor-persistence-bridge.ts"; +import { readExecutorInitialCheckpoints } from "./executor-checkpoint-state.ts"; + +/** Executor-local capability views; cleanup revokes these views, never their shared channel. */ +export async function createExecutorRuntimeFacades(options: { + input: ExecutorRuntimeInstall; + channel: ExecutorChannel; + signal: AbortSignal; +}): Promise { + const { input, channel } = options; + const lifetime = new AbortController(); + const signal = AbortSignal.any([options.signal, channel.signal, lifetime.signal]); + const resolveModelRuntime = await createExecutorModelRuntimeResolver({ + channel, + allowedModelIds: new Set(input.grant.models.map((model) => model.id)), + signal, + }); + function cleanup(): Promise { + revokeModelRuntimeResolver(resolveModelRuntime); + lifetime.abort(); + return Promise.resolve(); + } + try { + signal.throwIfAborted(); + const hostIds = new Set(input.grant.hostToolFacadeIds); + const remoteIds = new Set(input.grant.remoteToolSourceIds); + const expected = new Set([...hostIds, ...remoteIds]); + if (expected.size !== hostIds.size + remoteIds.size) { + throw new TypeError("Ambiguous executor tool source grant"); + } + const sources = expected.size ? await createExecutorRemoteToolSources({ channel, signal }) : []; + if (sources.length !== expected.size || sources.some((source) => !expected.has(source.id))) { + throw new TypeError("Executor tool sources do not match installation"); + } + const hostTools = new Map(); + const remoteToolSources = new Map(); + for (const source of sources) { + if (remoteIds.has(source.id)) { + remoteToolSources.set(source.id, source); + continue; + } + const definitions = await source.listTools({ abortSignal: signal }); + const tools: HostToolSet = Object.create(null); + for (const definition of definitions) { + const alias = input.hostToolAliases?.find((entry) => + entry.sourceId === source.id && entry.toolName === definition.name + ); + tools[definition.name] = { + id: definition.name, + ...(alias ? { ownerAgentId: alias.ownerAgentId, shortName: alias.shortName } : {}), + title: definition.title, + description: definition.description, + inputSchemaJson: definition.parameters, + async execute(args: unknown, context?: ToolExecutionContext) { + signal.throwIfAborted(); + if (!args || typeof args !== "object" || Array.isArray(args)) { + throw new TypeError("Invalid executor tool arguments"); + } + return await source.executeTool( + definition.name, + args as Record, + context, + ); + }, + }; + } + hostTools.set(source.id, tools); + } + const persistence = createExecutorPersistenceFacades({ + ...await readExecutorInitialCheckpoints({ + channel, + signal, + capabilityIds: input.capabilities.persistence, + }), + channel, + signal, + capabilityIds: input.capabilities.persistence, + }); + const state = input.capabilities.projectSteering || input.capabilities.conversationUserText + ? (await import("./executor-state-bridge.ts")).createExecutorStateFacades({ + channel, + capabilityIds: { + projectSteering: input.capabilities.projectSteering, + conversationUserText: input.capabilities.conversationUserText, + }, + agentId: input.grant.agentId, + projectId: input.grant.execution.projectId, + branchId: input.grant.execution.branchId, + signal, + }) + : {}; + signal.throwIfAborted(); + return { resolveModelRuntime, hostTools, remoteToolSources, ...persistence, ...state, cleanup }; + } catch (error) { + await cleanup(); + throw error; + } +} diff --git a/src/agent/hosted/executor-runtime-install-schema.ts b/src/agent/hosted/executor-runtime-install-schema.ts new file mode 100644 index 0000000000..88d56db29b --- /dev/null +++ b/src/agent/hosted/executor-runtime-install-schema.ts @@ -0,0 +1,101 @@ +import type { InferSchema, Schema, SchemaValidator } from "#veryfront/extensions/schema/index.ts"; +import { defineSchema } from "#veryfront/schemas/index.ts"; +import { snapshotBoundedJsonValue } from "#veryfront/schemas/json-value.ts"; +import { getExecutorBindingSchema } from "../executor/protocol.ts"; +import { + getHostedExecutorOwnerSchema, + getHostedExecutorSourceSchema, +} from "./executor-session-schema.ts"; +import { getExecutorRuntimeGrantDataSchema } from "./executor-runtime-prepare-schema.ts"; +import { getExecutorPersistenceCapabilityIdsSchema } from "./executor-persistence-schema.ts"; +import { getExecutorDiscoveryIdSchema } from "./executor-discovery-schema.ts"; +import { getExecutorToolIdSchema } from "./executor-tool-schema.ts"; + +function artifactShape(v: SchemaValidator) { + return { + version: v.literal(1), + owner: getHostedExecutorOwnerSchema(), + source: getHostedExecutorSourceSchema(), + root: v.literal("project"), + }; +} + +/** Image-builder-owned metadata outside the project payload. No caller-selected paths. */ +export const getExecutorArtifactManifestSchema = defineSchema((v) => + v.object(artifactShape(v)).strict() +); + +export const getExecutorRuntimeInstallSchema = defineSchema((v) => + v.object({ + ...artifactShape(v), + binding: getExecutorBindingSchema(), + grant: getExecutorRuntimeGrantDataSchema(), + /** Trusted ownership metadata for selected host tools, independent of source listings. */ + hostToolAliases: v.array( + v.object({ + sourceId: getExecutorToolIdSchema(), + toolName: getExecutorToolIdSchema(), + ownerAgentId: getExecutorDiscoveryIdSchema(), + shortName: getExecutorToolIdSchema(), + }).strict(), + ).max(4096).optional(), + capabilities: v.object({ + persistence: getExecutorPersistenceCapabilityIdsSchema(), + projectSteering: getExecutorDiscoveryIdSchema().optional(), + conversationUserText: getExecutorDiscoveryIdSchema().optional(), + }).strict(), + }).strict().refine(({ grant, capabilities, hostToolAliases }) => { + const aliases = new Set(); + for (const alias of hostToolAliases ?? []) { + const key = JSON.stringify([alias.sourceId, alias.toolName]); + if ( + aliases.has(key) || alias.ownerAgentId !== grant.agentId || + !grant.hostToolFacadeIds.includes(alias.sourceId) || + !grant.allowedToolNames.includes(alias.toolName) + ) return false; + aliases.add(key); + } + const execution = grant.execution; + if ( + (execution.projectId !== null || grant.requiredCapabilities?.includes("project-steering")) && + !capabilities.projectSteering + ) return false; + if ( + grant.requiredCapabilities?.includes("conversation-user-text") && + !capabilities.conversationUserText + ) return false; + if (execution.kind === "canonical") { + if ( + !capabilities.persistence.publishParentRunEvents || + !capabilities.persistence.toolExposureCheckpoint + ) return false; + if ( + execution.providerReplay === "required" && + !capabilities.persistence.providerReplayCheckpoint + ) return false; + } + return new Set(grant.models.map((model) => model.id)).size === grant.models.length && + grant.models.some((model) => model.id === grant.defaultModelId); + }, "Missing or ambiguous executor installation authority") +); + +export type ExecutorArtifactManifest = InferSchema< + ReturnType +>; +export type ExecutorRuntimeInstall = InferSchema< + ReturnType +>; + +/** Snapshot before parsing so installation never retains a caller's mutable objects. */ +export function parseExecutorInstallation(schema: Schema, input: unknown): T { + const snapshot = snapshotBoundedJsonValue(input); + if ( + !snapshot.success || + new TextEncoder().encode(JSON.stringify(snapshot.value)).byteLength > 64 * 1024 + ) { + throw new TypeError("Invalid executor installation"); + } + const result = schema.safeParse(snapshot.value); + if (!result.success) throw new TypeError("Invalid executor installation"); + return result.data; +} diff --git a/src/agent/hosted/executor-runtime-install.test.ts b/src/agent/hosted/executor-runtime-install.test.ts new file mode 100644 index 0000000000..5a9c4ec42f --- /dev/null +++ b/src/agent/hosted/executor-runtime-install.test.ts @@ -0,0 +1,270 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import type { ExecutorOperation, ExecutorOperationContext } from "../executor/channel.ts"; +import type { JsonValue } from "#veryfront/schemas/index.ts"; +import { createExecutorRuntimeInstallation } from "./executor-runtime-install.ts"; +import { + getExecutorRuntimeInstallSchema, + parseExecutorInstallation, +} from "./executor-runtime-install-schema.ts"; +import { assertThrows } from "#veryfront/testing/assert.ts"; + +const binding = { allocationId: "allocation", invocationId: "invocation", generation: 1 }; +const artifact = { + version: 1, + owner: { scopeKind: "global", serviceName: "veryfront-agent" }, + source: { type: "release", releaseId: "release-1" }, + root: "project", +} as const; +const grant = { + agentId: "coder", + defaultModelId: "model", + maxSteps: 3, + models: [{ id: "model", maxOutputTokens: 100, providerToolNames: [] }], + allowedToolNames: [], + hostToolFacadeIds: [], + remoteToolSourceIds: [], + execution: { kind: "ephemeral", projectId: null }, +} as const; +function request(): JsonValue { + return JSON.parse( + JSON.stringify({ ...artifact, binding, grant, capabilities: { persistence: {} } }), + ); +} +function context(signal = new AbortController().signal): ExecutorOperationContext { + return { binding, signal, deadline: Date.now() + 10_000 }; +} +async function call( + operations: ReadonlyMap, + name: string, + input: JsonValue, + ctx = context(), +) { + const operation = operations.get(name); + if (operation?.mode !== "unary") throw new Error("Missing unary operation"); + return await operation.handle(input, ctx); +} +function runtime() { + const ended = Promise.withResolvers(); + let closes = 0; + return { + operations: new Map([ + ["discovery.describe", { mode: "unary", handle: () => ({ discovered: true }) }], + ["agent.describe", { mode: "unary", handle: () => ({ described: true }) }], + ["runtime.prepare", { mode: "unary", handle: () => ({ prepared: true }) }], + ["agent.stream", { + mode: "stream", + async *handle() { + yield { streamed: true }; + }, + }], + ]), + settled: ended.promise, + close: () => { + closes++; + ended.resolve(); + return Promise.resolve(); + }, + get closes() { + return closes; + }, + }; +} + +describe("executor runtime installation", () => { + it("accepts host aliases only for the installed owner, source, and canonical tool", () => { + const alias = { + sourceId: "host", + toolName: "owned-paper", + ownerAgentId: "coder", + shortName: "fetch-paper", + }; + const input = { + ...artifact, + binding, + grant: { ...grant, allowedToolNames: ["owned-paper"], hostToolFacadeIds: ["host"] }, + capabilities: { persistence: {} }, + hostToolAliases: [alias], + }; + const parsed = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), input); + assertEquals(parsed.hostToolAliases, [alias]); + for ( + const aliases of [ + [alias, alias], + [{ ...alias, sourceId: "other" }], + [{ ...alias, toolName: "fetch-paper" }], + [{ ...alias, ownerAgentId: "other" }], + [{ ...alias, shortName: "" }], + ] + ) { + assertThrows(() => + parseExecutorInstallation(getExecutorRuntimeInstallSchema(), { + ...input, + hostToolAliases: aliases, + }) + ); + } + }); + + it("registers fixed dispatch before bootstrap snapshots it, and imports only after authenticated installation", async () => { + let imports = 0; + const loaded = runtime(); + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => { + imports++; + return Promise.resolve(loaded); + }, + }); + const operations = new Map(owner.operations); + await assertRejects(() => call(operations, "discovery.describe", {})); + assertEquals(imports, 0); + assertEquals(await call(operations, "runtime.install", request()), { installed: true }); + assertEquals(imports, 1); + assertEquals(await call(operations, "discovery.describe", {}), { discovered: true }); + await assertRejects(() => call(operations, "runtime.install", request())); + await owner.close(); + await owner.settled; + assertEquals(loaded.closes, 1); + }); + + it("rejects changed owner, source, root, binding and undeclared fields before project imports", async () => { + let imports = 0; + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => { + imports++; + return Promise.resolve(runtime()); + }, + }); + for ( + const change of [ + { owner: { scopeKind: "global", serviceName: "other" } }, + { source: { type: "release", releaseId: "other" } }, + { root: "/arbitrary" }, + { binding: { ...binding, generation: 2 } }, + { providerToken: "synthetic-marker" }, + ] + ) { + await assertRejects(() => + call(owner.operations, "runtime.install", { + ...JSON.parse(JSON.stringify(request())), + ...change, + }) + ); + } + await assertRejects(() => + call(owner.operations, "runtime.install", request(), { + ...context(), + binding: { ...binding, invocationId: "other" }, + }) + ); + assertEquals(imports, 0); + await owner.close(); + }); + + it("requires canonical persistence and project capabilities before importing", async () => { + let imports = 0; + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => { + imports++; + return Promise.resolve(runtime()); + }, + }); + const input = JSON.parse(JSON.stringify(request())); + input.grant.execution = { + kind: "canonical", + projectId: "project-1", + conversationId: "conversation", + runId: "run", + messageId: "message", + providerReplay: "required", + }; + await assertRejects(() => call(owner.operations, "runtime.install", input)); + input.capabilities = { + persistence: { + publishParentRunEvents: "events", + toolExposureCheckpoint: "tools", + providerReplayCheckpoint: "replay", + }, + }; + await assertRejects(() => call(owner.operations, "runtime.install", input)); + assertEquals(imports, 0); + input.capabilities.projectSteering = "steering"; + assertEquals(await call(owner.operations, "runtime.install", input), { installed: true }); + await owner.close(); + }); + + it("revokes dispatch immediately while retaining late setup and cleanup until settled", async () => { + const pending = Promise.withResolvers>(); + const loaded = runtime(); + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => pending.promise, + }); + const installation = call(owner.operations, "runtime.install", request()); + await assertRejects(() => call(owner.operations, "runtime.install", request())); + const closing = owner.close(); + let settled = false; + void owner.settled.then(() => { + settled = true; + }); + await Promise.resolve(); + assertEquals(settled, false); + await assertRejects(() => call(owner.operations, "discovery.describe", {})); + pending.resolve(loaded); + await assertRejects(() => installation); + await closing; + await owner.settled; + assertEquals(loaded.closes, 1); + }); + + it("does not initialize when the installation request is already cancelled", async () => { + let imports = 0; + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => { + imports++; + return Promise.resolve(runtime()); + }, + }); + await assertRejects(() => + call(owner.operations, "runtime.install", request(), context(AbortSignal.abort())) + ); + assertEquals(imports, 0); + await owner.close(); + }); + + it("retains an outstanding dispatched operation even after runtime cleanup acknowledges closure", async () => { + const pending = Promise.withResolvers(); + const loaded = runtime(); + loaded.operations.set("discovery.describe", { mode: "unary", handle: () => pending.promise }); + const owner = createExecutorRuntimeInstallation({ + binding, + artifact, + install: () => Promise.resolve(loaded), + }); + await call(owner.operations, "runtime.install", request()); + const discovery = call(owner.operations, "discovery.describe", {}); + const closing = owner.close(); + let settled = false; + void owner.settled.then(() => { + settled = true; + }); + await loaded.settled; + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(settled, false); + pending.resolve({ discovered: true }); + await discovery; + await closing; + await owner.settled; + assertEquals(settled, true); + }); +}); diff --git a/src/agent/hosted/executor-runtime-install.ts b/src/agent/hosted/executor-runtime-install.ts new file mode 100644 index 0000000000..30274e4568 --- /dev/null +++ b/src/agent/hosted/executor-runtime-install.ts @@ -0,0 +1,178 @@ +import type { ExecutorOperation, ExecutorOperationContext } from "../executor/channel.ts"; +import { type ExecutorBinding, getExecutorBindingSchema } from "../executor/protocol.ts"; +import { sameHostedExecutorOwner } from "./executor-session-schema.ts"; +import { verifyHostedRuntimeSourceBinding } from "./runtime-source-binding.ts"; +import { + type ExecutorArtifactManifest, + type ExecutorRuntimeInstall, + getExecutorArtifactManifestSchema, + getExecutorRuntimeInstallSchema, + parseExecutorInstallation, +} from "./executor-runtime-install-schema.ts"; + +export interface InstalledExecutorRuntime { + operations: ReadonlyMap; + close(): Promise; + readonly settled: Promise; +} + +const operationModes = { + "discovery.describe": "unary", + "agent.describe": "unary", + "runtime.prepare": "unary", + "agent.stream": "stream", +} as const; + +/** + * Executor-only installation gate. Register its fixed map before starting the + * authenticated bootstrap; project discovery belongs exclusively in install(). + * The broker remains authoritative for grants and operation phases. + */ +export function createExecutorRuntimeInstallation(options: { + binding: ExecutorBinding; + artifact: ExecutorArtifactManifest; + signal?: AbortSignal; + install(input: ExecutorRuntimeInstall, signal: AbortSignal): Promise; +}) { + const binding = parseExecutorInstallation(getExecutorBindingSchema(), options.binding); + const artifact = parseExecutorInstallation(getExecutorArtifactManifestSchema(), options.artifact); + const lifetime = new AbortController(); + const settled = Promise.withResolvers(); + void settled.promise.catch(() => {}); + let phase: "empty" | "installing" | "installed" | "closed" = "empty"; + let setup: Promise | undefined; + let dispatch: ReadonlyMap | undefined; + let closing: Promise | undefined; + const tasks = new Set>(); + + function retainOperation() { + const task = Promise.withResolvers(); + tasks.add(task.promise); + return () => { + tasks.delete(task.promise); + task.resolve(); + }; + } + + function sameBinding(value: ExecutorBinding) { + return binding.allocationId === value.allocationId && binding.generation === value.generation && + binding.invocationId === value.invocationId; + } + function assertActive(context: ExecutorOperationContext) { + if ( + phase === "closed" || context.signal.aborted || context.deadline <= Date.now() || + !sameBinding(context.binding) + ) throw new Error("Executor installation unavailable"); + } + function close(): Promise { + if (closing) return closing; + phase = "closed"; + dispatch = undefined; + // Memoize before abort listeners can reenter. Retain even non-cooperative + // setup and runtime work until actual settlement, not cancellation notice. + closing = Promise.resolve().then(async () => { + let loaded: InstalledExecutorRuntime | undefined; + try { + loaded = await setup; + } catch { /* Failed setup owns its partial resources. */ } + if (loaded) { + const results = await Promise.allSettled([ + Promise.resolve().then(() => loaded.close()), + loaded.settled, + ]); + await Promise.allSettled([...tasks]); + if (results.some((result) => result.status === "rejected")) { + throw new Error("Executor installation cleanup failed"); + } + } + }); + lifetime.abort(new Error("Executor installation closed")); + options.signal?.removeEventListener("abort", abort); + void closing.then(settled.resolve, settled.reject); + return closing; + } + function abort() { + void close().catch(() => {}); + } + options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) abort(); + + const operations = new Map(); + operations.set("runtime.install", { + mode: "unary", + async handle(value, context) { + assertActive(context); + if (phase !== "empty") throw new Error("Executor runtime already installed"); + const input = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), value); + if ( + !sameBinding(input.binding) || !sameHostedExecutorOwner(input.owner, artifact.owner) || + verifyHostedRuntimeSourceBinding(artifact.source, input.source) !== undefined || + input.root !== artifact.root + ) throw new Error("Executor installation not granted"); + phase = "installing"; + context.signal.addEventListener("abort", abort, { once: true }); + setup = Promise.resolve().then(() => { + assertActive(context); + return options.install(input, lifetime.signal); + }); + try { + const loaded = await setup; + assertActive(context); + const registered = new Map(loaded.operations); + for (const [name, mode] of Object.entries(operationModes)) { + if (registered.get(name)?.mode !== mode) throw new Error("Incomplete executor runtime"); + } + dispatch = registered; + phase = "installed"; + return { installed: true }; + } catch { + void close().catch(() => {}); + throw new Error("Executor installation failed"); + } finally { + context.signal.removeEventListener("abort", abort); + } + }, + }); + for (const [name, mode] of Object.entries(operationModes)) { + if (mode === "unary") { + operations.set(name, { + mode, + async handle(value, context) { + assertActive(context); + const operation = dispatch?.get(name); + if (phase !== "installed" || operation?.mode !== "unary") { + throw new Error("Executor runtime not installed"); + } + const release = retainOperation(); + try { + return await operation.handle(value, { + ...context, + signal: AbortSignal.any([context.signal, lifetime.signal]), + }); + } finally { + release(); + } + }, + }); + } else {operations.set(name, { + mode, + async *handle(value, context) { + assertActive(context); + const operation = dispatch?.get(name); + if (phase !== "installed" || operation?.mode !== "stream") { + throw new Error("Executor runtime not installed"); + } + const release = retainOperation(); + try { + yield* operation.handle(value, { + ...context, + signal: AbortSignal.any([context.signal, lifetime.signal]), + }); + } finally { + release(); + } + }, + });} + } + return { operations, close, settled: settled.promise, signal: lifetime.signal }; +} diff --git a/src/agent/hosted/executor-runtime-prepare-schema.ts b/src/agent/hosted/executor-runtime-prepare-schema.ts index 8ac4c8be54..22f4c81416 100644 --- a/src/agent/hosted/executor-runtime-prepare-schema.ts +++ b/src/agent/hosted/executor-runtime-prepare-schema.ts @@ -3,6 +3,11 @@ import type { InferSchema, Schema } from "#veryfront/extensions/schema/index.ts" import { defineSchema, getJsonValueSchema } from "#veryfront/schemas/index.ts"; import { defineError, VeryfrontError } from "#veryfront/errors/types.ts"; import { getExecutorDiscoveryIdSchema } from "#veryfront/agent/hosted/executor-discovery-schema.ts"; +import { + getExecutorAgentFailureCodeSchema, + getExecutorPreparedRuntimeHandleSchema, +} from "#veryfront/agent/hosted/executor-agent-schema.ts"; + import { getRuntimeAgentMarkdownDefinitionSchema } from "#veryfront/agent/runtime/agent-definition.ts"; const failureStatus = { @@ -16,6 +21,19 @@ const failureStatus = { EXECUTOR_RUNTIME_CLOSED: 410, ABORTED: 499, } as const; +type RuntimePreparationFailureCode = keyof typeof failureStatus; +const runtimePreparationFailureCodes = Object.keys(failureStatus) as [ + RuntimePreparationFailureCode, + ...RuntimePreparationFailureCode[], +]; +const getExecutorRuntimePreparationFailureCodeSchema = defineSchema((v) => + v.union([v.enum(runtimePreparationFailureCodes), getExecutorAgentFailureCodeSchema()]) +); +export function isExecutorRuntimePreparationFailureCode( + value: unknown, +): value is RuntimePreparationFailureCode { + return runtimePreparationFailureCodes.includes(value as RuntimePreparationFailureCode); +} export class ExecutorRuntimePreparationError extends VeryfrontError { constructor(readonly code: keyof typeof failureStatus) { super( @@ -60,6 +78,26 @@ export type ExecutorRuntimePrepareRequest = InferSchema< ReturnType >; +export const getExecutorRuntimePrepareResultSchema = defineSchema((v) => + v.discriminatedUnion("ok", [ + v.object({ + ok: v.literal(true), + value: v.object({ + preparedRuntimeHandle: getExecutorPreparedRuntimeHandleSchema(), + runtimeKind: v.literal("framework"), + modelId: getExecutorDiscoveryIdSchema(), + }).strict(), + }).strict(), + v.object({ + ok: v.literal(false), + code: getExecutorRuntimePreparationFailureCodeSchema(), + }).strict(), + ]) +); +export type ExecutorRuntimePrepareResult = InferSchema< + ReturnType +>; + export const getExecutorRuntimeSteeringSchema = defineSchema((v) => v.object({ agent: getRuntimeAgentMarkdownDefinitionSchema(), diff --git a/src/agent/hosted/executor-runtime-prepare.test.ts b/src/agent/hosted/executor-runtime-prepare.test.ts index 9a80a1c890..8ec67cec19 100644 --- a/src/agent/hosted/executor-runtime-prepare.test.ts +++ b/src/agent/hosted/executor-runtime-prepare.test.ts @@ -32,6 +32,10 @@ import { } from "#veryfront/agent/hosted/executor-runtime-prepare-schema.ts"; import { createExecutorChannel } from "#veryfront/agent/executor/channel.ts"; import { createExecutorHostedChatRuntimeAgent } from "./executor-agent-bridge.ts"; +import { createExecutorModelRuntimeResolver } from "./executor-model-bridge.ts"; +import { createEphemeralHostedExecutorModelBroker } from "./executor-model-dispatch.ts"; +import { scriptedModel } from "../runtime/model-runtime.test-helpers.ts"; +import { executorAgentJson } from "./executor-agent-schema.ts"; const binding = { allocationId: "prepare-allocation", @@ -160,6 +164,46 @@ async function prepare( } describe("executor runtime preparation", () => { + for ( + const request of [ + { thinking: { enabled: true, budgetTokens: 8192 } }, + { thinking: { enabled: true, budgetTokens: 4096 }, maxOutputTokens: 4097 }, + ] + ) { + it("rejects a completion allowance exhausted or exceeded by thinking before streaming", async () => { + const id = "veryfront-cloud/anthropic/claude-sonnet-4-6"; + const f = fixture({ + grant: { + ...grant, + defaultModelId: id, + models: new Map([[id, { maxOutputTokens: 8192, providerToolNames: [] }]]), + }, + config: { model: id }, + facades: { + resolveModelRuntime: () => ({ + ...model, + provider: "anthropic", + modelId: "claude-sonnet-4-6", + }), + }, + }); + try { + assertEquals( + await prepare( + f.owner, + executorAgentJson({ agentId: "coder", ...request }, "EXECUTOR_AGENT_INPUT_TOO_LARGE"), + ), + { + ok: false, + code: "EXECUTOR_RUNTIME_NOT_GRANTED", + }, + ); + } finally { + await f.owner.close(); + } + }); + } + for (const selection of [undefined, [], ["load_skill"]]) { it(`normalizes implicit disabled skill tools while retaining explicit rejection (${JSON.stringify(selection)})`, async () => { const f = fixture({ @@ -184,6 +228,100 @@ describe("executor runtime preparation", () => { }); } + for ( + const scenario of [ + { name: "catalog thinking", model: "claude-sonnet-4-6", expected: 6144 }, + { + name: "explicit thinking", + model: "claude-sonnet-4-6", + thinking: { enabled: true, budgetTokens: 4096 }, + expected: 4096, + }, + { + name: "default enabled thinking", + model: "claude-sonnet-4-6", + thinking: { enabled: true }, + expected: 4096, + }, + { + name: "disabled thinking", + model: "claude-sonnet-4-6", + thinking: { enabled: false }, + expected: 8192, + }, + { name: "adaptive thinking", model: "claude-opus-4-8", expected: 8192 }, + { name: "explicit completion", model: "claude-sonnet-4-6", completion: 512, expected: 512 }, + ] as const + ) { + it(`fits ${scenario.name} inside the broker's total output allowance`, async () => { + const id = `veryfront-cloud/anthropic/${scenario.model}`; + const provider = scriptedModel([{ text: "Complete" }], { + provider: "anthropic", + modelId: scenario.model, + only: "stream", + }); + const forward = new TransformStream(); + const backward = new TransformStream(); + const broker = createExecutorChannel({ + binding, + transport: { readable: forward.readable, writable: backward.writable }, + operations: createEphemeralHostedExecutorModelBroker({ + resolveModelRuntime: () => provider, + allowedModelIds: new Set([id]), + scope: { binding, signal: new AbortController().signal, assertActive() {} }, + grant: { + maxCalls: 2, + maxConcurrentCalls: 1, + models: new Map([[id, { maxOutputTokens: 8192, providerTools: [] }]]), + }, + prepared: { conversationId: null, canonicalRootRun: null }, + }), + }); + const executor = createExecutorChannel({ + binding, + transport: { readable: backward.readable, writable: forward.writable }, + }); + const resolveModelRuntime = await createExecutorModelRuntimeResolver({ + channel: executor, + allowedModelIds: new Set([id]), + }); + const f = fixture({ + grant: { + ...grant, + defaultModelId: id, + models: new Map([[id, { maxOutputTokens: 8192, providerToolNames: [] }]]), + }, + config: { model: id }, + facades: { resolveModelRuntime }, + }); + try { + const events = await Array.fromAsync( + await preparedStream( + f, + executorAgentJson({ + agentId: "coder", + ...("thinking" in scenario ? { thinking: scenario.thinking } : {}), + ...("completion" in scenario ? { maxOutputTokens: scenario.completion } : {}), + }, "EXECUTOR_AGENT_INPUT_TOO_LARGE"), + ), + ); + assertEquals(provider.callCount, 1); + assertEquals(provider.calls[0]?.maxOutputTokens, scenario.expected); + assert( + events.some((event) => + event !== null && typeof event === "object" && !Array.isArray(event) && + event.type === "complete" + ), + ); + } finally { + await f.owner.close(); + broker.close(); + executor.close(); + await Promise.all([broker.settled, executor.settled]); + } + }); + } + it("keeps skill references and scripts outside a loader-only grant after loading a skill", async () => { const visible: string[][] = []; const f = fixture({ @@ -1881,6 +2019,7 @@ Synthetic source instructions.`, it(`refreshes steering after ${path} only on a successful steering change (${success})`, async () => { let calls = 0; let refreshes = 0; + let refreshedTools: readonly string[] | undefined; const systems: string[] = []; const f = fixture({ grant: { @@ -1892,8 +2031,9 @@ Synthetic source instructions.`, facades: { projectSteering: { prepare: ({ definition }) => Promise.resolve({ agent: definition }), - refresh: () => { + refresh: (_signal, availableToolNames) => { refreshes++; + refreshedTools = availableToolNames; return "Updated synthetic steering"; }, }, @@ -1923,6 +2063,7 @@ Synthetic source instructions.`, await Array.fromAsync(await preparedStream(f)); assertEquals(calls, 2); assertEquals(refreshes, expectedRefreshes); + assertEquals(refreshedTools, expectedRefreshes === 1 ? ["update_file"] : undefined); assertEquals(systems[1]?.includes("Updated synthetic steering"), expectedRefreshes === 1); } finally { await f.owner.close(); @@ -1930,6 +2071,70 @@ Synthetic source instructions.`, }); } + for (const extraToolCount of [0, 128]) { + it(`refreshes steering with the model-visible provider-compatible tools (${extraToolCount} extra tools)`, async () => { + const remoteNames = [ + "update_file", + ...Array.from({ length: extraToolCount }, (_, index) => `zz_tool_${index}`), + ]; + const visible: string[][] = []; + let refreshedTools: readonly string[] | undefined; + const f = fixture({ + config: { providerTools: ["web_search"] }, + grant: { + ...grant, + models: new Map([[modelId, { maxOutputTokens: 200, providerToolNames: ["web_search"] }]]), + allowedToolNames: remoteNames, + remoteToolSourceIds: ["api"], + execution: { kind: "ephemeral", projectId: "synthetic-project" }, + }, + facades: { + projectSteering: { + prepare: ({ definition }) => Promise.resolve({ agent: definition }), + refresh: (_signal, availableToolNames) => { + refreshedTools = availableToolNames; + return "Updated synthetic steering"; + }, + }, + remoteToolSources: new Map([["api", { + id: "api", + listTools: () => + Promise.resolve(remoteNames.map((name) => ({ + ...syntheticRemoteTool(name), + parameters: { + type: "object", + properties: { path: { type: "string" }, project_reference: { type: "string" } }, + required: ["project_reference"], + }, + }))), + executeTool: () => Promise.resolve({ success: true }), + }]]), + resolveModelRuntime: () => ({ + ...model, + doStream: (options) => { + visible.push( + (options as ModelRuntimeCallOptions).tools?.map((tool) => tool.name) ?? [], + ); + return finishStream(visible.length === 1 ? "update_file" : undefined, { + path: "AGENTS.md", + }); + }, + }), + }, + }); + try { + await Array.fromAsync(await preparedStream(f)); + assertEquals(visible.length, 2); + assertEquals(visible[0]!.length, Math.min(extraToolCount + 2, 128)); + assert(visible[0]!.includes("web_search")); + assertEquals(visible[1], visible[0]); + assertEquals([...(refreshedTools ?? [])].sort(), [...visible[1]!].sort()); + } finally { + await f.owner.close(); + } + }); + } + it("reserves the catalog thinking budget when the request omits thinking and output limits", async () => { const selectedModel = "veryfront-cloud/anthropic/claude-sonnet-4-6"; let captured: ModelRuntimeCallOptions | undefined; diff --git a/src/agent/hosted/executor-runtime-prepare.ts b/src/agent/hosted/executor-runtime-prepare.ts index 3632305a1f..899c1d48aa 100644 --- a/src/agent/hosted/executor-runtime-prepare.ts +++ b/src/agent/hosted/executor-runtime-prepare.ts @@ -181,7 +181,10 @@ export interface ExecutorRuntimeFacades { signal: AbortSignal; }, ): Promise>; - refresh(signal: AbortSignal): Promise | AgentSystem; + refresh( + signal: AbortSignal, + availableToolNames?: readonly string[], + ): Promise | AgentSystem; }; latestConversationUserText?: (signal: AbortSignal) => Promise; publishParentRunEvents?: NonNullable; @@ -749,7 +752,7 @@ export function createExecutorRuntimePreparation(input: Options) { modelId, sourceIntegrationPolicy: runtime.sourceIntegrationPolicy, refreshSystem: facades.projectSteering - ? () => facades.projectSteering!.refresh(streamSignal) + ? () => facades.projectSteering!.refresh(streamSignal, toolAssembly.modelVisibleToolNames) : undefined, }; const runtimeOptions: NonNullable[1]> = { diff --git a/src/agent/hosted/executor-runtime-settlement.test.ts b/src/agent/hosted/executor-runtime-settlement.test.ts new file mode 100644 index 0000000000..afd2b2dd42 --- /dev/null +++ b/src/agent/hosted/executor-runtime-settlement.test.ts @@ -0,0 +1,22 @@ +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; +import { awaitExecutorCleanup } from "./executor-runtime-settlement.ts"; + +it("retains channel retirement after runtime cleanup fails", async () => { + const channel = Promise.withResolvers(); + let settled = false; + const closing = awaitExecutorCleanup([ + Promise.reject(new Error("Synthetic cleanup failure")), + channel.promise, + ]); + void closing.then(() => { + settled = true; + }, () => { + settled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(settled, false); + channel.resolve(); + await assertRejects(() => closing); + assertEquals(settled, true); +}); diff --git a/src/agent/hosted/executor-runtime-settlement.ts b/src/agent/hosted/executor-runtime-settlement.ts new file mode 100644 index 0000000000..7ad3f9b7c2 --- /dev/null +++ b/src/agent/hosted/executor-runtime-settlement.ts @@ -0,0 +1,7 @@ +/** Join the executor runtime and channel retirement branches. */ +export async function awaitExecutorCleanup(tasks: readonly Promise[]): Promise { + const results = await Promise.allSettled(tasks); + if (results.some((result) => result.status === "rejected")) { + throw new Error("Executor cleanup failed"); + } +} diff --git a/src/agent/hosted/executor-session-pool.test.ts b/src/agent/hosted/executor-session-pool.test.ts index 0720dcbecb..d916cf1d14 100644 --- a/src/agent/hosted/executor-session-pool.test.ts +++ b/src/agent/hosted/executor-session-pool.test.ts @@ -27,6 +27,9 @@ function sessionDouble() { binding: undefined, accepted: false, accept() {}, + runOwned(operation) { + return operation(); + }, close() { closeCalls++; controller.abort(); diff --git a/src/agent/hosted/executor-session.test.ts b/src/agent/hosted/executor-session.test.ts index 9f91928c23..91b241dca6 100644 --- a/src/agent/hosted/executor-session.test.ts +++ b/src/agent/hosted/executor-session.test.ts @@ -1,5 +1,11 @@ import "#veryfront/schemas/_test-setup.ts"; -import { assert, assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { + assert, + assertEquals, + assertRejects, + assertStrictEquals, + assertThrows, +} from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { createExecutorChannel, type ExecutorChannel } from "../executor/channel.ts"; import { ManualMonotonicClock } from "../streaming/lifecycle/testing.ts"; @@ -792,6 +798,55 @@ describe("hosted executor session", () => { } }); + it("retains broker-owned work through bounded close until the original promise settles", async () => { + const work = Promise.withResolvers(); + const f = fixture(); + const session = f.start(); + await session.ready; + const owned = session.runOwned(() => work.promise); + const closing = session.close(); + f.time.advanceBy(500); + assertEquals((await closing).release, "released"); + let settled = false; + void session.settled.then(() => settled = true); + await tick(); + assertEquals(settled, false); + work.resolve("persisted"); + assertEquals(await owned, "persisted"); + await session.settled; + assertEquals(settled, true); + await f.peerClosed(); + }); + + it("propagates owned-work rejection and retires it from settlement", async () => { + const f = fixture(); + const session = f.start(); + await session.ready; + const failure = new Error("synthetic owned failure"); + const rejected = await assertRejects(() => session.runOwned(() => Promise.reject(failure))); + assertStrictEquals(rejected, failure); + await session.close(); + await session.settled; + await f.peerClosed(); + }); + + it("rejects owned work after close without invoking the thunk", async () => { + const f = fixture(); + const session = f.start(); + await session.ready; + await session.close(); + let invoked = false; + await assertRejects(() => + session.runOwned(() => { + invoked = true; + return Promise.resolve(); + }) + ); + assertEquals(invoked, false); + await session.settled; + await f.peerClosed(); + }); + it("retains session admission through noncooperative incoming operation cleanup", async () => { const finish = Promise.withResolvers(); const started = Promise.withResolvers(); diff --git a/src/agent/hosted/executor-session.ts b/src/agent/hosted/executor-session.ts index 128d52403e..471234ee54 100644 --- a/src/agent/hosted/executor-session.ts +++ b/src/agent/hosted/executor-session.ts @@ -82,6 +82,9 @@ export interface HostedExecutorSessionCloseResult { release: "not-allocated" | "released" | "reaper-required"; } +/** Retain broker-local asynchronous work through complete session settlement. */ +export type HostedExecutorOwnedWork = (operation: () => Promise) => Promise; + export interface HostedExecutorSession { /** TLS and invocation-channel readiness, before remote runtime preparation/acceptance. */ readonly ready: Promise; @@ -94,6 +97,11 @@ export interface HostedExecutorSession { readonly accepted: boolean; /** Call after remote preparation. Execution ownership detaches the preparation request before 202. */ accept(ownership: { kind: "request" } | { kind: "execution"; signal?: AbortSignal }): void; + /** + * Retain broker-local work outside channel handlers until its original promise settles. + * Owned work must not await this session's own close or settled promise. + */ + runOwned(operation: () => Promise): Promise; close(reason?: "completed" | "canceled"): Promise; } @@ -273,6 +281,14 @@ class Session implements HostedExecutorSession { this.#cancelTimer("preparation"); } + async runOwned(operation: () => Promise): Promise { + this.#assertActive(); + if (typeof operation !== "function") { + throw new TypeError("Executor owned work requires an operation"); + } + return await this.#track(operation); + } + close(reason: "completed" | "canceled" = "canceled"): Promise { this.#stop(reason); return this.closed; diff --git a/src/agent/hosted/executor-state-bridge.test.ts b/src/agent/hosted/executor-state-bridge.test.ts new file mode 100644 index 0000000000..3334c697c1 --- /dev/null +++ b/src/agent/hosted/executor-state-bridge.test.ts @@ -0,0 +1,299 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { createExecutorChannel, type ExecutorOperation } from "../executor/channel.ts"; +import type { ExecutorRuntimeFacades } from "./executor-runtime-prepare.ts"; +import { createExecutorStateBroker, createExecutorStateFacades } from "./executor-state-bridge.ts"; +import { executorStateJson, executorStateOperations } from "./executor-state-schema.ts"; + +const binding = { + allocationId: "allocation-state", + generation: 3, + invocationId: "invocation-state", +}; +const scope = { agentId: "coder", projectId: "project-1", branchId: "branch-1" }; +const capabilityIds = { + projectSteering: "steering-capability", + conversationUserText: "text-capability", +}; +const definition = { id: "coder", name: "Coder", description: "Codes", instructions: "Work" }; +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function pair(operations: ReadonlyMap) { + const forward = new TransformStream(); + const backward = new TransformStream(); + const executor = createExecutorChannel({ + binding, + transport: { readable: backward.readable, writable: forward.writable }, + }); + const broker = createExecutorChannel({ + binding, + transport: { readable: forward.readable, writable: backward.writable }, + operations, + }); + return { + executor, + broker, + async close() { + executor.close(); + await broker.closed; + await Promise.all([executor.settled, broker.settled]); + }, + }; +} + +describe("executor state bridge", () => { + it("forwards a narrowed tool selection and rejects names outside the installed grant", async () => { + const seen: unknown[] = []; + const channels = pair(createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + allowedToolNames: ["read_file", "load_skill"], + prepareProjectSteering: async (input) => ({ agent: input.definition }), + refreshProjectSteering: async (_signal, availableToolNames) => { + seen.push(availableToolNames); + return "refresh"; + }, + })); + try { + const facades = createExecutorStateFacades({ + channel: channels.executor, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + }); + const signal = new AbortController().signal; + await facades.projectSteering!.refresh(signal, ["read_file"]); + await facades.projectSteering!.refresh(signal, ["load_skill"]); + await facades.projectSteering!.refresh(signal); + await assertRejects(() => facades.projectSteering!.refresh(signal, ["write_file"])); + assertEquals(seen, [["read_file"], ["load_skill"], []]); + } finally { + await channels.close(); + } + }); + + it("uses broker-owned scope and returns bounded steering and conversation state", async () => { + const seen: unknown[] = []; + const channels = pair(createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds, + prepareProjectSteering: async (input) => { + seen.push({ + kind: "prepare", + agentId: input.definition.id, + projectId: input.projectId, + branchId: input.branchId, + }); + return { + agent: input.definition, + initialProjectInstructions: "Project instructions", + initialSkills: [], + }; + }, + refreshProjectSteering: async () => { + seen.push({ kind: "refresh" }); + return "Refreshed instructions"; + }, + latestConversationUserText: async () => { + seen.push({ kind: "text" }); + return "Latest user text"; + }, + })); + try { + const facades = createExecutorStateFacades({ + channel: channels.executor, + ...scope, + capabilityIds, + }); + const compatible: Pick< + ExecutorRuntimeFacades, + "projectSteering" | "latestConversationUserText" + > = facades; + assertEquals(compatible, facades); + assertEquals( + await facades.projectSteering?.prepare({ + ...scope, + definition, + signal: new AbortController().signal, + }), + { + agent: definition, + initialProjectInstructions: "Project instructions", + initialSkills: [], + }, + ); + assertEquals( + await facades.projectSteering?.refresh(new AbortController().signal), + "Refreshed instructions", + ); + assertEquals( + await facades.latestConversationUserText?.(new AbortController().signal), + "Latest user text", + ); + assertEquals(seen, [ + { kind: "prepare", agentId: "coder", projectId: "project-1", branchId: "branch-1" }, + { kind: "refresh" }, + { kind: "text" }, + ]); + } finally { + await channels.close(); + } + }); + + it("rejects executor scope changes before channel dispatch", async () => { + let calls = 0; + const channels = pair(createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + prepareProjectSteering: async (input) => ({ agent: input.definition }), + refreshProjectSteering: async () => { + calls++; + return "refresh"; + }, + })); + try { + const facades = createExecutorStateFacades({ + channel: channels.executor, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + }); + await assertRejects(() => + facades.projectSteering!.prepare({ + definition, + projectId: "project-2", + branchId: "branch-1", + signal: new AbortController().signal, + }) + ); + await assertRejects(() => + facades.projectSteering!.prepare({ + definition: { ...definition, id: "other" }, + projectId: "project-1", + branchId: "branch-1", + signal: new AbortController().signal, + }) + ); + assertEquals(calls, 0); + } finally { + await channels.close(); + } + }); + + it("binds broker handlers and rejects wire authority fields", async () => { + let calls = 0; + const operations = createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { conversationUserText: capabilityIds.conversationUserText }, + latestConversationUserText: async () => { + calls++; + return null; + }, + }); + const operation = operations.get(executorStateOperations.latestConversationUserText); + if (operation?.mode !== "unary") throw new Error("missing operation"); + for ( + const value of [ + { capabilityId: capabilityIds.conversationUserText, authToken: "secret" }, + { capabilityId: capabilityIds.conversationUserText, projectId: "project-2" }, + ] + ) { + await assertRejects(() => + Promise.resolve( + operation.handle(executorStateJson(value), { + binding, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }), + ) + ); + } + await assertRejects(() => + Promise.resolve( + operation.handle({ capabilityId: capabilityIds.conversationUserText }, { + binding: { ...binding, generation: 4 }, + signal: new AbortController().signal, + deadline: Date.now() + 1_000, + }), + ) + ); + assertEquals(calls, 0); + }); + + it("fails closed on incomplete capability handlers", () => { + assertThrows(() => + createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + }) + ); + }); + + it("forwards refresh cancellation and retains a noncooperative read through channel settlement", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const channels = pair(createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + prepareProjectSteering: async (input) => ({ agent: input.definition }), + refreshProjectSteering: async (signal) => { + entered.resolve(signal); + await release.promise; + return "late refresh"; + }, + })); + const owner = new AbortController(); + const call = new AbortController(); + const facades = createExecutorStateFacades({ + channel: channels.executor, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + signal: owner.signal, + }); + const pending = facades.projectSteering!.refresh(call.signal); + const observed = await entered.promise; + call.abort(); + await tick(); + assertEquals(observed.aborted, true); + + channels.executor.close(); + await channels.broker.closed; + await assertRejects(() => pending); + let settled = false; + void channels.broker.settled.then(() => settled = true); + await tick(); + assertEquals(settled, false); + release.resolve(); + await Promise.all([channels.executor.settled, channels.broker.settled]); + assertEquals(settled, true); + }); + + it("rejects provider transport authority in refreshed system data", async () => { + const channels = pair(createExecutorStateBroker({ + expectedBinding: binding, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + prepareProjectSteering: async (input) => ({ agent: input.definition }), + refreshProjectSteering: async () => [{ + role: "system", + content: "Synthetic", + providerOptions: { headers: { authorization: "secret" } }, + }], + })); + try { + const facades = createExecutorStateFacades({ + channel: channels.executor, + ...scope, + capabilityIds: { projectSteering: capabilityIds.projectSteering }, + }); + await assertRejects(() => facades.projectSteering!.refresh(new AbortController().signal)); + } finally { + await channels.close(); + } + }); +}); diff --git a/src/agent/hosted/executor-state-bridge.ts b/src/agent/hosted/executor-state-bridge.ts new file mode 100644 index 0000000000..2793440b01 --- /dev/null +++ b/src/agent/hosted/executor-state-bridge.ts @@ -0,0 +1,262 @@ +import type { AgentSystem } from "#veryfront/agent/types.ts"; +import type { RuntimeAgentMarkdownDefinition } from "#veryfront/agent/runtime/agent-definition.ts"; +import type { HostedChatRuntimeProjectSteering } from "./chat-runtime-contract.ts"; +import type { ExecutorBinding } from "../executor/protocol.ts"; +import { getExecutorBindingSchema } from "../executor/protocol.ts"; +import type { + ExecutorChannel, + ExecutorOperation, + ExecutorOperationContext, +} from "../executor/channel.ts"; +import { + type ExecutorStateCapabilityIds, + executorStateJson, + executorStateOperations, + getExecutorAgentSystemSchema, + getExecutorConversationUserTextResultSchema, + getExecutorProjectSteeringPrepareRequestSchema, + getExecutorProjectSteeringRefreshRequestSchema, + getExecutorProjectSteeringResultSchema, + getExecutorStateCapabilityIdsSchema, + getExecutorStateReadRequestSchema, + parseExecutorStateData, +} from "./executor-state-schema.ts"; +import { getExecutorDiscoveryIdSchema } from "./executor-discovery-schema.ts"; + +export type { ExecutorStateCapabilityIds } from "./executor-state-schema.ts"; + +type Scope = { agentId: string; projectId: string | null; branchId?: string | null }; +type ProjectSteeringPrepareInput = { + definition: RuntimeAgentMarkdownDefinition; + projectId: string | null; + branchId?: string | null; + signal: AbortSignal; +}; +export interface ExecutorStateFacades { + projectSteering?: { + prepare(input: ProjectSteeringPrepareInput): Promise< + HostedChatRuntimeProjectSteering + >; + refresh(signal: AbortSignal, availableToolNames?: readonly string[]): Promise; + }; + latestConversationUserText?: (signal: AbortSignal) => Promise; +} + +function sameBinding(left: Readonly, right: Readonly) { + return left.allocationId === right.allocationId && left.generation === right.generation && + left.invocationId === right.invocationId; +} +function parseScope(value: Scope): Scope { + return Object.freeze({ + agentId: parseExecutorStateData(getExecutorDiscoveryIdSchema(), value.agentId), + projectId: value.projectId === null + ? null + : parseExecutorStateData(getExecutorDiscoveryIdSchema(), value.projectId), + ...(value.branchId === undefined ? {} : { + branchId: value.branchId === null + ? null + : parseExecutorStateData(getExecutorDiscoveryIdSchema(), value.branchId), + }), + }); +} +function authorize( + context: ExecutorOperationContext, + expectedBinding: ExecutorBinding, + actual: string, + expected: string, +) { + if (!sameBinding(context.binding, expectedBinding) || actual !== expected) { + throw new TypeError("Managed state operation is not authorized"); + } +} + +export function createExecutorStateBroker( + options: Scope & { + expectedBinding: ExecutorBinding; + capabilityIds: ExecutorStateCapabilityIds; + /** Installed tool grant used to authorize the executor's narrowed refresh selection. */ + allowedToolNames?: readonly string[]; + prepareProjectSteering?: ( + input: ProjectSteeringPrepareInput, + ) => Promise>; + refreshProjectSteering?: ( + signal: AbortSignal, + availableToolNames?: readonly string[], + ) => Promise | AgentSystem; + latestConversationUserText?: NonNullable; + }, +): ReadonlyMap { + const expectedBinding = Object.freeze(getExecutorBindingSchema().parse(options.expectedBinding)); + const scope = parseScope(options); + const allowedToolNames = new Set(options.allowedToolNames ?? []); + const capabilityIds = Object.freeze( + parseExecutorStateData(getExecutorStateCapabilityIdsSchema(), options.capabilityIds), + ); + const hasSteering = options.prepareProjectSteering !== undefined && + options.refreshProjectSteering !== undefined; + if ( + (capabilityIds.projectSteering !== undefined) !== hasSteering || + (!!options.prepareProjectSteering !== !!options.refreshProjectSteering) + ) throw new TypeError("Managed state capability configuration is incomplete"); + if ( + (capabilityIds.conversationUserText === undefined) !== + (options.latestConversationUserText === undefined) + ) throw new TypeError("Managed state capability configuration is incomplete"); + const operations = new Map(); + let steeringTail = Promise.resolve(undefined); + const scheduleSteering = ( + context: ExecutorOperationContext, + work: () => Promise | T, + ): Promise => { + const current = steeringTail.then(() => { + context.signal.throwIfAborted(); + return work(); + }); + steeringTail = current.catch(() => {}); + return current; + }; + if ( + capabilityIds.projectSteering && options.prepareProjectSteering && + options.refreshProjectSteering + ) { + const capabilityId = capabilityIds.projectSteering; + const prepare = options.prepareProjectSteering; + const refresh = options.refreshProjectSteering; + operations.set(executorStateOperations.prepareProjectSteering, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorStateData( + getExecutorProjectSteeringPrepareRequestSchema(), + value, + ); + authorize(context, expectedBinding, request.capabilityId, capabilityId); + if (request.definition.id !== scope.agentId) { + throw new TypeError("Managed state agent is not authorized"); + } + const result = await scheduleSteering( + context, + () => prepare({ ...scope, definition: request.definition, signal: context.signal }), + ); + const parsed = parseExecutorStateData( + getExecutorProjectSteeringResultSchema(), + executorStateJson(result), + ); + if (parsed.agent.id !== scope.agentId) { + throw new TypeError("Managed state result is not authorized"); + } + return executorStateJson(parsed); + }, + }); + operations.set(executorStateOperations.refreshProjectSteering, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorStateData( + getExecutorProjectSteeringRefreshRequestSchema(), + value, + ); + authorize(context, expectedBinding, request.capabilityId, capabilityId); + const availableToolNames = request.availableToolNames ?? []; + if (availableToolNames.some((name) => !allowedToolNames.has(name))) { + throw new TypeError("Managed state tool selection is not authorized"); + } + const result = await scheduleSteering( + context, + () => refresh(context.signal, availableToolNames), + ); + return executorStateJson( + parseExecutorStateData(getExecutorAgentSystemSchema(), executorStateJson(result)), + ); + }, + }); + } + if (capabilityIds.conversationUserText && options.latestConversationUserText) { + const capabilityId = capabilityIds.conversationUserText; + const read = options.latestConversationUserText; + operations.set(executorStateOperations.latestConversationUserText, { + mode: "unary", + async handle(value, context) { + const request = parseExecutorStateData(getExecutorStateReadRequestSchema(), value); + authorize(context, expectedBinding, request.capabilityId, capabilityId); + const text = await read(context.signal); + return executorStateJson( + parseExecutorStateData(getExecutorConversationUserTextResultSchema(), { text }), + ); + }, + }); + } + return operations; +} + +export function createExecutorStateFacades( + options: Scope & { + channel: ExecutorChannel; + capabilityIds: ExecutorStateCapabilityIds; + signal?: AbortSignal; + }, +): ExecutorStateFacades { + const scope = parseScope(options); + const capabilityIds = Object.freeze( + parseExecutorStateData(getExecutorStateCapabilityIdsSchema(), options.capabilityIds), + ); + return { + ...(capabilityIds.projectSteering + ? { + projectSteering: { + async prepare(input: ProjectSteeringPrepareInput) { + if ( + scope.projectId !== input.projectId || scope.branchId !== input.branchId || + input.definition.id !== scope.agentId + ) { + throw new TypeError("Managed state scope cannot change"); + } + const request = executorStateJson({ + capabilityId: capabilityIds.projectSteering, + definition: input.definition, + }); + parseExecutorStateData(getExecutorProjectSteeringPrepareRequestSchema(), request); + const signal = options.signal + ? AbortSignal.any([options.signal, input.signal]) + : input.signal; + const result = await options.channel.request( + executorStateOperations.prepareProjectSteering, + request, + { signal }, + ); + const parsed = parseExecutorStateData(getExecutorProjectSteeringResultSchema(), result); + if (parsed.agent.id !== scope.agentId) { + throw new TypeError( + "Managed state result is not authorized", + ); + } + return parsed; + }, + async refresh(signal: AbortSignal, availableToolNames?: readonly string[]) { + const combined = options.signal ? AbortSignal.any([options.signal, signal]) : signal; + const result = await options.channel.request( + executorStateOperations.refreshProjectSteering, + { + capabilityId: capabilityIds.projectSteering!, + availableToolNames: [...availableToolNames ?? []], + }, + { signal: combined }, + ); + return parseExecutorStateData(getExecutorAgentSystemSchema(), result); + }, + }, + } + : {}), + ...(capabilityIds.conversationUserText + ? { + latestConversationUserText: async (signal: AbortSignal) => { + const combined = options.signal ? AbortSignal.any([options.signal, signal]) : signal; + const result = await options.channel.request( + executorStateOperations.latestConversationUserText, + { capabilityId: capabilityIds.conversationUserText! }, + { signal: combined }, + ); + return parseExecutorStateData(getExecutorConversationUserTextResultSchema(), result).text; + }, + } + : {}), + }; +} diff --git a/src/agent/hosted/executor-state-schema.ts b/src/agent/hosted/executor-state-schema.ts new file mode 100644 index 0000000000..62ed5b7716 --- /dev/null +++ b/src/agent/hosted/executor-state-schema.ts @@ -0,0 +1,154 @@ +import type { InferSchema, Schema } from "#veryfront/extensions/schema/index.ts"; +import { defineSchema, getJsonValueSchema, type JsonValue } from "#veryfront/schemas/index.ts"; +import { snapshotBoundedJsonValue } from "#veryfront/schemas/json-value.ts"; +import { + getExecutorAgentDefinitionSchema, + getExecutorDiscoveryIdSchema, +} from "./executor-discovery-schema.ts"; +import { EXECUTOR_MAX_FRAME_BYTES } from "../executor/protocol.ts"; + +const MAX_STATE_PAYLOAD_BYTES = EXECUTOR_MAX_FRAME_BYTES - 2_048; +const MAX_SKILLS = 128; +const MAX_ARRAY_ITEMS = 3_000; +const MAX_TEXT_LENGTH = 1_048_576; +const encoder = new TextEncoder(); +const forbiddenProviderFields = new Set([ + "headers", + "authorization", + "auth", + "apikey", + "apitoken", + "authtoken", + "credential", + "credentials", + "baseurl", + "url", + "endpoint", + "fetch", + "signal", + "abortsignal", +]); + +export const executorStateOperations = Object.freeze( + { + prepareProjectSteering: "state.project-steering.prepare", + refreshProjectSteering: "state.project-steering.refresh", + latestConversationUserText: "state.conversation-user-text", + } as const, +); + +export const getExecutorStateCapabilityIdsSchema = defineSchema((v) => + v.object({ + projectSteering: getExecutorDiscoveryIdSchema().optional(), + conversationUserText: getExecutorDiscoveryIdSchema().optional(), + }).strict().refine((value) => { + const ids = Object.values(value).filter((id): id is string => id !== undefined); + return new Set(ids).size === ids.length; + }, "Managed state capability IDs must be distinct") +); +export type ExecutorStateCapabilityIds = InferSchema< + ReturnType +>; + +const getCapabilityRequestSchema = defineSchema((v) => + v.object({ capabilityId: getExecutorDiscoveryIdSchema() }).strict() +); +export const getExecutorProjectSteeringPrepareRequestSchema = defineSchema((_v) => + getCapabilityRequestSchema().extend({ definition: getExecutorAgentDefinitionSchema() }).strict() +); +export const getExecutorStateReadRequestSchema = getCapabilityRequestSchema; +export const getExecutorProjectSteeringRefreshRequestSchema = defineSchema((v) => + getCapabilityRequestSchema().extend({ + availableToolNames: v.array(getExecutorDiscoveryIdSchema()).max(1_000).optional(), + }).strict() +); + +const getSkillSelectorPolicySchema = defineSchema((v) => + v.discriminatedUnion("kind", [ + v.object({ kind: v.literal("all-visible"), source: v.enum(["omitted", "true"] as const) }) + .strict(), + v.object({ kind: v.literal("none") }).strict(), + v.object({ + kind: v.literal("allowlist"), + entries: v.array(getExecutorDiscoveryIdSchema()).max(1_000), + }).strict(), + ]) +); +const getRuntimeSkillDefinitionSchema = defineSchema((v) => + v.object({ + id: getExecutorDiscoveryIdSchema(), + name: v.string().min(1).max(256), + displayName: v.string().max(256).optional(), + description: v.string().max(MAX_TEXT_LENGTH), + instructions: v.string().max(MAX_TEXT_LENGTH), + allowedTools: v.array(v.string().min(1).max(256)).max(100).optional(), + metadata: v.record(v.string().max(256), v.string().max(4_096)).optional(), + model: v.string().min(1).max(256).optional(), + thinking: v.union([v.literal(false), v.number().int().positive().max(1_000_000)]).optional(), + maxSteps: v.number().int().positive().max(1_000).optional(), + references: v.array(v.string().min(1).max(4_096)).max(MAX_ARRAY_ITEMS).optional(), + ownerAgentId: getExecutorDiscoveryIdSchema().optional(), + shortName: v.string().min(1).max(256).optional(), + sourcePath: v.string().min(1).max(4_096).optional(), + }).strict() +); +export const getExecutorProjectSteeringResultSchema = defineSchema((v) => + v.object({ + agent: getExecutorAgentDefinitionSchema(), + skillSelectorPolicy: getSkillSelectorPolicySchema().optional(), + environmentContext: v.string().max(MAX_TEXT_LENGTH).optional(), + initialProjectInstructions: v.string().max(MAX_TEXT_LENGTH).optional(), + initialSkills: v.array(getRuntimeSkillDefinitionSchema()).max(MAX_SKILLS).optional(), + }).strict() +); + +const getProviderOptionsSchema = defineSchema((v) => + v.record(v.string().max(256), getJsonValueSchema()).refine((value) => { + for (const field of Object.keys(value)) { + if (forbiddenProviderFields.has(field.replace(/[-_]/g, "").toLowerCase())) return false; + } + for (const bucket of Object.values(value)) { + if (bucket === null || typeof bucket !== "object" || Array.isArray(bucket)) continue; + for (const field of Object.keys(bucket)) { + if (forbiddenProviderFields.has(field.replace(/[-_]/g, "").toLowerCase())) return false; + } + } + return true; + }, "Managed state provider transport fields are forbidden") +); +export const getExecutorAgentSystemSchema = defineSchema((v) => + v.union([ + v.string().max(MAX_TEXT_LENGTH), + v.array( + v.object({ + role: v.literal("system"), + content: v.string().max(MAX_TEXT_LENGTH), + providerOptions: getProviderOptionsSchema().optional(), + }).strict(), + ).max(256), + ]) +); +export const getExecutorConversationUserTextResultSchema = defineSchema((v) => + v.object({ text: v.string().max(MAX_TEXT_LENGTH).nullable() }).strict() +); + +export function parseExecutorStateData(schema: Schema, value: unknown): T { + const result = schema.safeParse(value); + if (!result.success) throw new TypeError("Invalid managed state data"); + return result.data; +} + +export function executorStateJson(value: unknown): JsonValue { + let encoded: string | undefined; + try { + encoded = JSON.stringify(value); + } catch { + throw new TypeError("Invalid managed state data"); + } + if (encoded === undefined || encoder.encode(encoded).byteLength > MAX_STATE_PAYLOAD_BYTES) { + throw new TypeError("Invalid managed state data"); + } + const snapshot = snapshotBoundedJsonValue(JSON.parse(encoded)); + if (!snapshot.success) throw new TypeError("Invalid managed state data"); + return snapshot.value; +} diff --git a/src/agent/hosted/managed-broker-persistence.ts b/src/agent/hosted/managed-broker-persistence.ts new file mode 100644 index 0000000000..75fd4298bd --- /dev/null +++ b/src/agent/hosted/managed-broker-persistence.ts @@ -0,0 +1,222 @@ +import type { ChatMessageMetadata, ChatUiMessageChunk } from "#veryfront/chat/protocol.ts"; +import type { ConversationRunEvent } from "../conversation/run-events.ts"; +import { + type ConversationRunProjection, + getConversationRunProjectionSchema, +} from "../conversation/durable-contracts.ts"; +import { + createConversationHostedTerminalAdapter, + resolveConversationHostedStreamErrorState, +} from "../conversation/hosted-terminal.ts"; +import { createDurableRunEventSink } from "./durable-run-event-sink.ts"; +import { + createHostedConversationRunChunkMirrorFromCapability, + createHostedRunEventWriterCapability, +} from "./child-run-event-writer-token.ts"; +import { + createToolExposureCheckpointEvent, + type ToolExposureCheckpoint, +} from "../runtime/tool-exposure.ts"; +import { + createProviderReplayCheckpointEvent, + type ProviderReplayCheckpoint, +} from "../runtime/provider-replay.ts"; +import type { AgentRunEventSink } from "#veryfront/runtime/model-call-context.ts"; +import type { HostedLifecycleTerminalState } from "./lifecycle.ts"; +import type { HostedExecutorOwnedWork } from "./executor-session.ts"; + +/** Acknowledging output writes and terminal finalization for a canonical run. */ +export interface ManagedBrokerOutput { + write(chunk: ChatUiMessageChunk): Promise; + finish(input: { + completed: boolean; + error?: unknown; + metadata?: HostedLifecycleTerminalState["metadata"]; + }): Promise; +} + +/** Create exact-run API persistence callbacks while retaining credentials in the broker. */ +export function createManagedBrokerPersistence(input: { + apiUrl: string; + runEventToken: string; + run: ConversationRunProjection; + modelId: string; + resolveProvider(modelId: string): string; + fetch?: typeof globalThis.fetch; +}) { + const run = getConversationRunProjectionSchema().parse(input.run); + if (run.status !== "pending" && run.status !== "running" && run.status !== "waiting_for_tool") { + throw new TypeError("Managed broker persistence requires an active run"); + } + let sessionOwnedWork: HostedExecutorOwnedWork | undefined; + let retainedPersistenceTail = Promise.resolve(); + let cleaned = false; + const runQueueFlush = (operation: () => Promise): Promise => { + const owner = sessionOwnedWork; + if (!owner) { + return Promise.reject(new TypeError("Managed broker persistence is not session-bound")); + } + const owned = owner(operation); + const settled = owned.then(() => undefined, () => undefined); + retainedPersistenceTail = Promise.all([retainedPersistenceTail, settled]).then(() => undefined); + return owned; + }; + const capability = createHostedRunEventWriterCapability({ + apiUrl: input.apiUrl, + runId: run.runId, + runEventAppendToken: input.runEventToken, + fetch: input.fetch, + }); + const mirror = createHostedConversationRunChunkMirrorFromCapability(capability, { + expectedRunId: run.runId, + conversationId: run.conversationId, + latestEventId: run.latestEventId, + latestExternalEventSequence: run.latestExternalEventSequence, + runQueueFlush, + }); + if (!mirror) throw new TypeError("Managed broker run-event capability is not bound"); + const durableMirror = mirror; + const terminal = createConversationHostedTerminalAdapter({ + authToken: input.runEventToken, + apiUrl: input.apiUrl, + run, + fallbackModelId: input.modelId, + resolveProvider: input.resolveProvider, + fetch: input.fetch, + }); + const durableSink = createDurableRunEventSink({ mirror: durableMirror }); + let tail = Promise.resolve(); + let failure: unknown; + let failed = false; + let finished = false; + + const queue = ( + operation: (priorFailure: { failed: boolean; error: unknown }) => Promise, + terminal = false, + ): Promise => { + if (cleaned) return Promise.reject(new TypeError("Managed broker persistence is closed")); + if (!sessionOwnedWork) { + return Promise.reject(new TypeError("Managed broker persistence is not session-bound")); + } + if (finished && !terminal) { + return Promise.reject(new TypeError("Managed broker persistence is finished")); + } + const current = tail.then(async () => { + const priorFailure = { failed, error: failure }; + if (priorFailure.failed && !terminal) throw priorFailure.error; + try { + const result = await operation(priorFailure); + if (priorFailure.failed) throw priorFailure.error; + return result; + } catch (error) { + if (priorFailure.failed) throw priorFailure.error; + failure = error; + failed = true; + throw error; + } + }); + tail = current.then(() => undefined, () => undefined); + return current; + }; + const flush = async () => { + const snapshot = await durableMirror.flush({ throwOnTimeoutRetry: true }); + if (snapshot.disabled || snapshot.pendingEventCount > 0 || snapshot.inFlight) { + throw new TypeError("Managed broker output was not durably persisted"); + } + }; + const persistEvents = (events: ConversationRunEvent[]) => + queue(async () => { + await durableMirror.appendEvents(events); + await flush(); + }); + const modelRunEventSink: AgentRunEventSink = (event) => + queue(async () => await durableSink(event)); + const output: ManagedBrokerOutput = { + write(chunk) { + if (finished) return Promise.reject(new TypeError("Managed broker output is finished")); + return queue(async () => { + await durableMirror.handleChunk(chunk); + await flush(); + }); + }, + finish(result) { + if (finished) return Promise.reject(new TypeError("Managed broker output is finished")); + if (!sessionOwnedWork) { + return Promise.reject(new TypeError("Managed broker persistence is not session-bound")); + } + if (result.completed && result.error !== undefined) { + return Promise.reject(new TypeError("Completed managed output cannot carry an error")); + } + finished = true; + return queue(async (priorFailure) => { + // Terminal reporting is an independent best-effort path: even a poisoned + // write tail or final drain must attempt a failed terminal update, while + // callers still receive the original persistence error. + let terminalFailure = priorFailure; + if (!terminalFailure.failed) { + try { + await flush(); + } catch (error) { + terminalFailure = { failed: true, error }; + } + } + try { + if (terminalFailure.failed) { + await terminal.dispatch( + { + ...resolveConversationHostedStreamErrorState(terminalFailure.error), + metadata: result.metadata, + }, + ); + } else if (result.completed) { + await terminal.dispatch({ status: "completed", metadata: result.metadata }); + } else if (result.error !== undefined) { + await terminal.dispatch({ + ...resolveConversationHostedStreamErrorState(result.error), + metadata: result.metadata, + }); + } else { + await terminal.dispatch({ + status: "cancelled", + terminalErrorCode: "ABORTED", + terminalErrorMessage: "Managed executor output was cancelled", + metadata: result.metadata, + }); + } + } catch (terminalDispatchError) { + if (terminalFailure.failed) throw terminalFailure.error; + throw terminalDispatchError; + } + if (terminalFailure.failed && !priorFailure.failed) throw terminalFailure.error; + }, true); + }, + }; + async function cleanup(): Promise { + if (cleaned) return; + cleaned = true; + await tail; + await retainedPersistenceTail; + durableMirror.dispose(); + } + function bindSessionOwnedWork(owner: HostedExecutorOwnedWork): void { + if (typeof owner !== "function") { + throw new TypeError("Managed broker persistence owner must be a function"); + } + if (cleaned) throw new TypeError("Managed broker persistence is closed"); + if (sessionOwnedWork) { + throw new TypeError("Managed broker persistence is already session-bound"); + } + sessionOwnedWork = owner; + } + return { + bindSessionOwnedWork, + modelRunEventSink, + publishParentRunEvents: persistEvents, + persistToolExposureCheckpoint: (checkpoint: ToolExposureCheckpoint) => + persistEvents([createToolExposureCheckpointEvent(checkpoint)]), + persistProviderReplayCheckpoint: (checkpoint: ProviderReplayCheckpoint) => + persistEvents([createProviderReplayCheckpointEvent(checkpoint)]), + output, + cleanup, + }; +} diff --git a/src/agent/hosted/managed-broker-project-state.ts b/src/agent/hosted/managed-broker-project-state.ts new file mode 100644 index 0000000000..a3d13f29f6 --- /dev/null +++ b/src/agent/hosted/managed-broker-project-state.ts @@ -0,0 +1,150 @@ +import type { AgentSystem } from "../types.ts"; +import type { RuntimeAgentMarkdownDefinition } from "../runtime/agent-definition.ts"; +import { + createStrictRuntimeProjectFilesClient, + type RuntimeGetProjectFileOptions, + type RuntimeProjectFilesApiOptions, +} from "../runtime/project-files-client.ts"; +import { + getRuntimeProjectInstructions, + getRuntimeProjectSkillCatalog, +} from "../runtime/project-skill-catalog.ts"; +import { + resolveRuntimeSkillSelectorSnapshotForAgent, + type RuntimeSkillDefinition, +} from "../runtime/skill-metadata.ts"; +import { + assertResolvedSkillSelector, + createNoneSkillSelectorSnapshot, +} from "#veryfront/skill/selector.ts"; +import type { SkillDocumentParserProvider } from "#veryfront/extensions/parser/skill-document-parser.ts"; +import { buildInteractiveVeryfrontCloudRuntimeInstructions } from "./cloud-runtime-system-messages.ts"; +import type { HostedChatRuntimeProjectSteering } from "./chat-runtime-contract.ts"; + +type Scope = { projectId: string | null; branchId?: string | null }; + +/** + * Create broker-owned project steering. Refreshes show skills only when the + * broker-validated effective tool selection includes `load_skill`. + */ +export function createManagedBrokerProjectState( + options: Scope & { + apiUrl: string | URL; + authToken: string; + agentId: string; + builtinSkills?: readonly RuntimeSkillDefinition[]; + skillDocumentParserProvider?: SkillDocumentParserProvider; + environmentContext?: string; + fetch?: (url: string, init: RequestInit) => Promise; + latestConversationUserText?: (signal: AbortSignal) => Promise; + }, +) { + if (!options.agentId || !options.authToken) { + throw new TypeError("Managed broker project state requires fixed identity and authorization"); + } + const projectId = options.projectId; + const branchId = options.branchId; + const authToken = options.authToken; + const projectClient = createStrictRuntimeProjectFilesClient({ + apiUrl: new URL(options.apiUrl).toString(), + fetch: options.fetch, + }); + const builtinSkills = [...options.builtinSkills ?? []]; + let definition: RuntimeAgentMarkdownDefinition | undefined; + const assertScope = (input: Scope) => { + if (input.projectId !== projectId || input.branchId !== branchId) { + throw new TypeError("Managed broker project state scope cannot change"); + } + }; + const fileReader = (signal: AbortSignal) => async (input: RuntimeGetProjectFileOptions) => { + assertScope(input); + if (input.authToken !== authToken || projectId === null) { + throw new TypeError("Managed broker project authorization cannot change"); + } + return await projectClient.getProjectFile({ ...input, signal }); + }; + const fileLister = (signal: AbortSignal) => async (input: RuntimeProjectFilesApiOptions) => { + assertScope(input); + if (input.authToken !== authToken || projectId === null) { + throw new TypeError("Managed broker project authorization cannot change"); + } + return await projectClient.getProjectFiles({ ...input, signal }); + }; + const load = async (signal: AbortSignal) => { + signal.throwIfAborted(); + if (projectId === null) return { instructions: "", skills: [] as RuntimeSkillDefinition[] }; + const lookup = { projectId, branchId, authToken }; + const [instructionsResult, skillsResult] = await Promise.allSettled([ + getRuntimeProjectInstructions({ ...lookup, getProjectFile: fileReader(signal) }), + getRuntimeProjectSkillCatalog({ + ...lookup, + getProjectFile: fileReader(signal), + getProjectFiles: fileLister(signal), + builtinSkills, + skillDocumentParserProvider: options.skillDocumentParserProvider, + }), + ]); + if (instructionsResult.status === "rejected") throw instructionsResult.reason; + if (skillsResult.status === "rejected") throw skillsResult.reason; + signal.throwIfAborted(); + return { instructions: instructionsResult.value, skills: skillsResult.value }; + }; + const select = (agent: RuntimeAgentMarkdownDefinition, skills: RuntimeSkillDefinition[]) => { + if (agent.skills === false) return createNoneSkillSelectorSnapshot(); + const selected = resolveRuntimeSkillSelectorSnapshotForAgent({ + skills, + agentId: agent.id, + selector: agent.skills, + }); + assertResolvedSkillSelector(selected); + return selected; + }; + return { + async prepareProjectSteering( + input: Scope & { + definition: RuntimeAgentMarkdownDefinition; + signal: AbortSignal; + }, + ): Promise> { + assertScope(input); + if (input.definition.id !== options.agentId) { + throw new TypeError("Managed broker project agent cannot change"); + } + const loaded = await load(input.signal); + const selected = select(input.definition, loaded.skills); + definition = structuredClone(input.definition); + return { + agent: structuredClone(input.definition), + skillSelectorPolicy: selected.policy, + ...(options.environmentContext ? { environmentContext: options.environmentContext } : {}), + ...(loaded.instructions ? { initialProjectInstructions: loaded.instructions } : {}), + ...(selected.definitions.length ? { initialSkills: selected.definitions } : {}), + }; + }, + async refreshProjectSteering( + signal: AbortSignal, + availableToolNames: readonly string[] = [], + ): Promise { + if (!definition) throw new TypeError("Managed broker project state is not prepared"); + const loaded = await load(signal); + const selected = select(definition, loaded.skills); + return buildInteractiveVeryfrontCloudRuntimeInstructions({ + agentConfig: definition, + projectId, + branchId, + instructions: loaded.instructions, + skills: availableToolNames.includes("load_skill") ? selected.definitions : [], + availableToolNames, + environmentContext: options.environmentContext, + }); + }, + ...(options.latestConversationUserText + ? { + latestConversationUserText: async (signal: AbortSignal) => { + signal.throwIfAborted(); + return await options.latestConversationUserText!(signal); + }, + } + : {}), + }; +} diff --git a/src/agent/hosted/managed-executor-broker.test.ts b/src/agent/hosted/managed-executor-broker.test.ts new file mode 100644 index 0000000000..b72a566c73 --- /dev/null +++ b/src/agent/hosted/managed-executor-broker.test.ts @@ -0,0 +1,1116 @@ +import { + type ExecutorRuntimeInstall, + getExecutorRuntimeInstallSchema, + parseExecutorInstallation, +} from "./executor-runtime-install-schema.ts"; +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { createExecutorChannel, type ExecutorOperation } from "../executor/channel.ts"; +import type { ModelRuntime } from "#veryfront/provider/types.ts"; +import type { JsonValue } from "#veryfront/schemas/index.ts"; +import type { HostedExecutorSessionOptions } from "./executor-session.ts"; +import type { HostedExecutorAllocation } from "./executor-session-schema.ts"; +import type { ExecutorNodeTransport } from "./executor-node-transport.ts"; +import { + createManagedExecutorBroker, + type ManagedExecutorStartInput, +} from "./managed-executor-broker.ts"; +import { getExecutorRuntimePrepareResultSchema } from "./executor-runtime-prepare-schema.ts"; +import { readExecutorInitialCheckpoints } from "./executor-checkpoint-state.ts"; +import { executorStateOperations } from "./executor-state-schema.ts"; +import { createManagedBrokerPersistence } from "./managed-broker-persistence.ts"; +import { FakeTime } from "#std/testing/time"; +import { agent } from "#veryfront/agent/factory.ts"; +import { scriptedModel } from "#veryfront/agent/runtime/model-runtime.test-helpers.ts"; +import { createExecutorRuntimeInstallation } from "./executor-runtime-install.ts"; +import { createExecutorRuntimeFacades } from "./executor-runtime-facades.ts"; +import { createExecutorDiscovery } from "./executor-discovery.ts"; +import { createExecutorRuntimePreparation } from "./executor-runtime-prepare.ts"; + +const modelId = "veryfront-cloud/openai/synthetic"; +const owner = { scopeKind: "project" as const, projectId: "project-test" }; +const source = { type: "release" as const, releaseId: "release-test" }; +const image = `registry.example.test/executor@sha256:${"a".repeat(64)}`; +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function runtimeModel(): ModelRuntime { + return { + specificationVersion: "v2", + provider: "veryfront-cloud", + modelId: "synthetic", + doGenerate: () => Promise.resolve({ content: [], finishReason: "stop", usage: {} }), + doStream: () => + Promise.resolve({ + stream: new ReadableStream({ + start(c) { + c.close(); + }, + }), + }), + }; +} + +function fixture( + options: { + completeStream?: boolean; + agentId?: string; + prepareFailure?: boolean; + prepareModelId?: string; + brokerReadWait?: boolean; + initialCheckpoint?: boolean; + allocationLifetimeMs?: number; + hardDeadlineMs?: number; + } = {}, +) { + const now = Date.now(); + const request = { + allocationId: crypto.randomUUID(), + invocationId: crypto.randomUUID(), + owner, + source, + requestedAt: now, + prepareDeadlineAt: now + 10_000, + hardDeadlineAt: now + (options.hardDeadlineMs ?? 60_000), + }; + const calls: string[] = []; + let peer: ReturnType | undefined; + let generation = 0; + let preparationDenied = false; + let executionAllowed = false; + let installed: ExecutorRuntimeInstall | undefined; + let initialCheckpointRead = false; + const preparation = new AbortController(); + const prepareEntered = Promise.withResolvers(); + const prepareRelease = Promise.withResolvers(); + const allocator: HostedExecutorSessionOptions["allocator"] = { + allocate() { + calls.push("allocate"); + generation = 1; + return Promise.resolve(allocation("ready")); + }, + observe() { + calls.push("observe"); + return Promise.resolve(allocation("ready")); + }, + renew() { + calls.push("renew"); + return Promise.resolve(allocation("ready")); + }, + release(_binding, reason) { + calls.push(`release:${reason}`); + return Promise.resolve(allocation("released", reason)); + }, + }; + function allocation( + phase: "ready" | "released", + reason?: "completed" | "canceled", + ): HostedExecutorAllocation { + return { + binding: { + allocationId: request.allocationId, + invocationId: request.invocationId, + owner, + source, + generation, + brokerInstanceId: "broker-test", + }, + phase, + expiresAt: now + (options.allocationLifetimeMs ?? 30_000), + ...(reason ? { reason } : {}), + ...(phase === "ready" + ? { + endpoint: { + address: "192.0.2.10", + port: 8081, + podUid: "pod-test", + nodeName: "node-test", + image, + channelAuthenticated: false, + }, + } + : {}), + }; + } + const session: ManagedExecutorStartInput["session"] = { + request, + expectedBrokerInstanceId: "broker-test", + expectedImage: image, + allocator, + preparationSignal: preparation.signal, + pollIntervalMs: 1_000, + requestTimeoutMs: 1_000, + cleanupTimeoutMs: 50, + connectTransport(input): Promise { + calls.push("connect"); + const outbound = new TransformStream(); + const inbound = new TransformStream(); + const operations = new Map([ + ["runtime.install", { + mode: "unary", + async handle(value) { + installed = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), value); + calls.push("install"); + try { + await peer!.request("model.generate", { modelId, options: { prompt: [] } }); + } catch { + preparationDenied = true; + } + if (options.initialCheckpoint) { + const state = await readExecutorInitialCheckpoints({ + channel: peer!, + capabilityIds: { toolExposureCheckpoint: "tool-checkpoint" }, + }); + initialCheckpointRead = state.initialToolExposureCheckpoint?.loadedToolNames[0] === + "search"; + } + return { installed: true }; + }, + }], + ["agent.describe", { + mode: "unary", + handle() { + calls.push("describe"); + return { + ok: true, + value: { + source, + definition: { + id: options.agentId ?? "coder", + name: "Coder", + description: "Codes", + instructions: "Work", + }, + }, + }; + }, + }], + ["runtime.prepare", { + mode: "unary", + async handle(): Promise { + calls.push("prepare"); + if (options.brokerReadWait) { + await peer!.request(executorStateOperations.latestConversationUserText, { + capabilityId: "conversation-text", + }); + } + if (options.prepareFailure) { + return { ok: false, code: "EXECUTOR_RUNTIME_PREPARATION_FAILED" }; + } + return { + ok: true, + value: { + preparedRuntimeHandle: "prepared-1", + runtimeKind: "framework", + modelId: options.prepareModelId ?? modelId, + }, + }; + }, + }], + ["agent.stream", { + mode: "stream", + async *handle(): AsyncGenerator { + calls.push("stream"); + await peer!.request("model.generate", { + modelId, + options: { prompt: [], maxOutputTokens: installed!.grant.models[0]!.maxOutputTokens }, + }); + executionAllowed = true; + yield { type: "ready" }; + if (options.completeStream) { + yield { type: "event", event: { type: "message-finish" } }; + yield { type: "complete" }; + return; + } + await new Promise(() => {}); + }, + }], + ]); + peer = createExecutorChannel({ + binding: input.binding, + transport: { readable: outbound.readable, writable: inbound.writable }, + operations, + }); + return Promise.resolve({ + readable: inbound.readable, + writable: outbound.writable, + close() { + peer?.close(); + }, + }); + }, + }; + const installation: ManagedExecutorStartInput["installation"] = { + version: 1, + owner, + source, + root: "project", + grant: { + agentId: "coder", + defaultModelId: modelId, + maxSteps: 5, + models: [{ id: modelId, maxOutputTokens: 100, providerToolNames: [] }], + allowedToolNames: [], + hostToolFacadeIds: [], + remoteToolSourceIds: [], + execution: { kind: "ephemeral", projectId: null }, + }, + capabilities: { persistence: {} }, + }; + const input: ManagedExecutorStartInput = { + session, + installation, + prepare: { agentId: "coder" }, + model: { + resolver: (id) => id === modelId ? runtimeModel() : undefined, + grant: { + maxCalls: 4, + maxConcurrentCalls: 1, + models: new Map([[modelId, { maxOutputTokens: 100, providerTools: [] }]]), + }, + }, + tools: { catalog: new Map(), sources: new Map(), maxCalls: 4, maxConcurrent: 1 }, + persistence: {}, + state: {}, + }; + if (options.initialCheckpoint) { + installation.capabilities.persistence.toolExposureCheckpoint = "tool-checkpoint"; + input.persistence = { + initialToolExposureCheckpoint: { version: 2, loadedToolNames: ["search"] }, + persistToolExposureCheckpoint: () => Promise.resolve(), + }; + } + if (options.brokerReadWait) { + installation.capabilities.conversationUserText = "conversation-text"; + input.state = { + latestConversationUserText: async () => { + prepareEntered.resolve(); + await prepareRelease.promise; + return "late text"; + }, + }; + } + return { + calls, + input, + preparation, + prepareEntered: prepareEntered.promise, + releasePrepare: prepareRelease.resolve, + get installed() { + return installed; + }, + get peer() { + return peer; + }, + get preparationDenied() { + return preparationDenied; + }, + get executionAllowed() { + return executionAllowed; + }, + get initialCheckpointRead() { + return initialCheckpointRead; + }, + }; +} + +function configureCanonical( + input: ManagedExecutorStartInput, + runEventSink: NonNullable, + bindSessionOwnedWork?: ManagedExecutorStartInput["bindSessionOwnedWork"], +): void { + input.installation.grant.execution = { + kind: "canonical", + projectId: null, + conversationId: "conversation-1", + runId: "run-1", + messageId: "message-1", + providerReplay: "disabled", + }; + input.installation.capabilities.persistence = { + publishParentRunEvents: "parent-events", + toolExposureCheckpoint: "tool-checkpoint", + }; + input.persistence = { + publishParentRunEvents: () => Promise.resolve(), + persistToolExposureCheckpoint: () => Promise.resolve(), + }; + input.model.runEventSink = runEventSink; + if (bindSessionOwnedWork) input.bindSessionOwnedWork = bindSessionOwnedWork; +} + +describe("managed executor broker", () => { + it("executes an owned host tool selected by its short alias through installed runtime facades", async () => { + const f = fixture(); + const model = scriptedModel([ + { toolCalls: [{ id: "call", name: "owned-paper", input: {} }] }, + { text: "Complete" }, + ], { only: "stream", modelId: "synthetic", provider: "openai" }); + const executions: string[] = []; + f.input.model.resolver = () => model; + f.input.installation.grant.allowedToolNames = ["fetch-paper"]; + f.input.installation.grant.hostToolFacadeIds = ["host"]; + f.input.tools.catalog = new Map([ + ["fetch-paper", {}], + ["owned-paper", { ownerAgentId: "coder", shortName: "fetch-paper" }], + ]); + f.input.tools.sources = new Map([["host", { + allowedToolNames: new Set(["owned-paper"]), + context: {}, + source: { + id: "host", + listTools: () => + Promise.resolve([{ + name: "owned-paper", + description: "Read a synthetic paper", + parameters: { type: "object", properties: {} }, + }]), + executeTool: (name) => { + executions.push(name); + return Promise.resolve({ text: "Synthetic paper" }); + }, + }, + }]]); + f.input.session.connectTransport = ({ binding }) => { + const outbound = new TransformStream(); + const inbound = new TransformStream(); + const installation = createExecutorRuntimeInstallation({ + binding, + artifact: { version: 1, owner, source, root: "project" }, + async install(input, signal) { + const facades = await createExecutorRuntimeFacades({ input, channel: peer, signal }); + const discovery = createExecutorDiscovery({ + binding, + source, + projectDir: "/synthetic-project", + signal, + backend: { + load: () => + Promise.resolve({ + agents: new Map([[ + "coder", + agent({ + id: "coder", + model: modelId, + system: "Synthetic instructions", + tools: { "fetch-paper": true }, + }), + ]]), + tools: new Map(), + skills: new Map(), + prompts: new Map(), + resources: new Map(), + workflows: new Map(), + tasks: new Map(), + schedules: new Map(), + webhooks: new Map(), + evals: new Map(), + errors: [], + sourceIntegrationPolicy: { schemaVersion: 1, mode: "unrestricted" }, + }), + cleanup: () => Promise.resolve(), + }, + }); + return createExecutorRuntimePreparation({ + binding, + source, + facades, + discovery, + grant: { + ...input.grant, + models: new Map(input.grant.models.map(({ id, ...policy }) => [id, policy])), + }, + }); + }, + }); + const peer = createExecutorChannel({ + binding, + transport: { readable: outbound.readable, writable: inbound.writable }, + operations: installation.operations, + }); + return Promise.resolve({ + readable: inbound.readable, + writable: outbound.writable, + async close() { + await installation.close(); + peer.close(); + await peer.settled; + }, + }); + }; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let runtime: Awaited> | undefined; + try { + runtime = await broker.start(f.input); + runtime.accept({ kind: "execution" }); + const stream = await runtime.agent.stream({ + messages: [{ + id: "user", + role: "user", + parts: [{ type: "text", text: "Read" }], + timestamp: 1, + }], + abortSignal: new AbortController().signal, + }); + const events = await Array.fromAsync(stream.toUIMessageStream()); + assertEquals(model.toolNames(), ["owned-paper"]); + assertEquals(executions, ["owned-paper"]); + assertEquals(model.callCount, 2); + assert(events.some((event) => event.type === "finish")); + } finally { + await runtime?.close(); + await broker.shutdown(); + await broker.settled; + } + }); + + for (const mismatch of ["output tokens", "provider tools", "tool allowlist", "tool source"]) { + it(`rejects a broker ${mismatch} grant broader than its installation before allocation`, async () => { + const f = fixture(); + if (mismatch === "output tokens") { + f.input.model.grant.models.get(modelId)!.maxOutputTokens = 101; + } else if (mismatch === "provider tools") { + f.input.model.grant.models.get(modelId)!.providerTools = [{ + type: "provider", + name: "web_search", + id: "openai.web_search", + args: {}, + }]; + } else { + if (mismatch === "tool source") f.input.installation.grant.allowedToolNames = ["ungranted"]; + else f.input.installation.grant.remoteToolSourceIds = ["synthetic"]; + f.input.tools.catalog = new Map([["ungranted", {}]]); + f.input.tools.sources = new Map([["synthetic", { + source: { + id: "synthetic", + listTools: () => Promise.resolve([]), + executeTool: () => Promise.resolve({ result: "unexpected" }), + }, + allowedToolNames: new Set(["ungranted"]), + context: {}, + }]]); + } + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let runtime: Awaited> | undefined; + try { + await assertRejects( + async () => { + runtime = await broker.start(f.input); + }, + TypeError, + "exceeds the installed", + ); + assertEquals(f.calls, []); + assertEquals(broker.active, 0); + } finally { + await runtime?.close(); + await broker.shutdown(); + await broker.settled; + } + }); + } + + it("applies narrower broker model grants to preparation and completed generation", async () => { + const f = fixture({ completeStream: true }); + f.input.installation.grant.models[0]!.maxOutputTokens = 200; + f.input.installation.grant.models[0]!.providerToolNames = ["web_search", "web_fetch"]; + f.input.model.grant.models.get(modelId)!.providerTools = [{ + type: "provider", + name: "web_search", + id: "openai.web_search", + args: {}, + }]; + f.input.installation.grant.allowedToolNames = ["inspect"]; + f.input.tools.catalog = new Map([["inspect", {}]]); + f.input.installation.grant.remoteToolSourceIds = ["synthetic"]; + f.input.tools.sources = new Map([["synthetic", { + source: { + id: "synthetic", + listTools: () => Promise.resolve([]), + executeTool: () => Promise.resolve({ result: "done" }), + }, + allowedToolNames: new Set(["inspect"]), + context: {}, + }]]); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let runtime: Awaited> | undefined; + try { + runtime = await broker.start(f.input); + runtime.accept({ kind: "execution" }); + const stream = await runtime.agent.stream({ + messages: [], + abortSignal: new AbortController().signal, + }); + await Array.fromAsync(stream.toUIMessageStream()); + assertEquals(f.installed!.grant.models[0]!.maxOutputTokens, 100); + assertEquals(f.installed!.grant.models[0]!.providerToolNames, ["web_search"]); + assertEquals(f.input.installation.grant.models[0]!.maxOutputTokens, 200); + assertEquals(f.executionAllowed, true); + } finally { + await runtime?.close(); + await broker.shutdown(); + await broker.settled; + } + }); + + for ( + const [agentId, selector, capabilityName, accepted] of [ + ["coder", "fetch-paper", "coder--fetch-paper", true], + ["coder", "fetch-paper", "owned-paper", true], + ["research.coder", "fetch-paper", "research_coder--fetch-paper", true], + ["coder", "fetch-paper", "writer--fetch-paper", false], + ["coder", "coder--fetch-paper", "writer--fetch-paper", false], + ["coder", "coder--fetch-paper", "fetch-paper", false], + ] as const + ) { + it(`${accepted ? "accepts" : "rejects"} capability ${capabilityName} for ${agentId}'s ${selector} grant`, async () => { + const f = fixture({ agentId }); + f.input.installation.grant.agentId = agentId; + f.input.prepare.agentId = agentId; + f.input.installation.grant.allowedToolNames = [selector]; + f.input.tools.catalog = new Map([ + ["fetch-paper", {}], + ["coder--fetch-paper", { ownerAgentId: "coder", shortName: "fetch-paper" }], + ["research_coder--fetch-paper", { + ownerAgentId: "research.coder", + shortName: "fetch-paper", + }], + ["writer--fetch-paper", { ownerAgentId: "writer", shortName: "fetch-paper" }], + ]); + if (capabilityName === "owned-paper") { + f.input.tools.catalog = new Map([ + ["fetch-paper", {}], + ["owned-paper", { ownerAgentId: "coder", shortName: "fetch-paper" }], + ]); + } + f.input.installation.grant.remoteToolSourceIds = ["synthetic"]; + f.input.tools.sources = new Map([["synthetic", { + source: { + id: "synthetic", + listTools: () => Promise.resolve([]), + executeTool: () => Promise.resolve({ result: "done" }), + }, + allowedToolNames: new Set([capabilityName]), + context: {}, + }]]); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let runtime: Awaited> | undefined; + try { + if (accepted) { + runtime = await broker.start(f.input); + assertEquals(f.calls.includes("allocate"), true); + assertEquals(f.installed!.grant.allowedToolNames, [capabilityName]); + } else { + await assertRejects( + async () => { + runtime = await broker.start(f.input); + }, + TypeError, + "exceeds the installed", + ); + assertEquals(f.calls, []); + } + } finally { + await runtime?.close(); + await broker.shutdown(); + await broker.settled; + } + }); + } + + it("rejects a global broker capability shadowed by an owned project-local tool", async () => { + const f = fixture(); + f.input.installation.grant.allowedToolNames = ["fetch-paper"]; + f.input.installation.grant.remoteToolSourceIds = ["synthetic"]; + f.input.tools.catalog = new Map([ + ["fetch-paper", {}], + ["owned-paper", { ownerAgentId: "coder", shortName: "fetch-paper" }], + ]); + f.input.tools.sources = new Map([["synthetic", { + source: { + id: "synthetic", + listTools: () => Promise.resolve([]), + executeTool: () => Promise.resolve({ result: "unexpected" }), + }, + allowedToolNames: new Set(["fetch-paper"]), + context: {}, + }]]); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let runtime: Awaited> | undefined; + try { + await assertRejects( + async () => { + runtime = await broker.start(f.input); + }, + TypeError, + "exceeds the installed", + ); + assertEquals(f.calls, []); + } finally { + await runtime?.close(); + await broker.shutdown(); + await broker.settled; + } + }); + + for (const selectOtherModel of [false, true]) { + it(`authorizes only the selected model's effective provider tools for steering (${selectOtherModel})`, async () => { + const otherModelId = "veryfront-cloud/anthropic/synthetic"; + const selectedModelId = selectOtherModel ? otherModelId : modelId; + const selectedTool = selectOtherModel ? "web_fetch" : "web_search"; + const rejectedTool = selectOtherModel ? "web_search" : "web_fetch"; + const f = fixture({ prepareModelId: selectedModelId }); + if (selectOtherModel) f.input.prepare.modelId = selectedModelId; + f.input.installation.grant.models = [modelId, otherModelId].map((id) => ({ + id, + maxOutputTokens: 100, + providerToolNames: ["web_search", "web_fetch"], + })); + f.input.model.grant.models = new Map([ + [modelId, { + maxOutputTokens: 100, + providerTools: [{ + type: "provider", + name: "web_search", + id: "openai.web_search", + args: {}, + }], + }], + [otherModelId, { + maxOutputTokens: 100, + providerTools: [{ + type: "provider", + name: "web_fetch", + id: "anthropic.web_fetch", + args: {}, + }], + }], + ]); + f.input.installation.capabilities.projectSteering = "steering"; + f.input.state.prepareProjectSteering = ({ definition }) => + Promise.resolve({ agent: definition }); + const selections: (readonly string[] | undefined)[] = []; + f.input.state.refreshProjectSteering = (_signal, names) => { + selections.push(names); + return Promise.resolve("Refreshed"); + }; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const runtime = await broker.start(f.input); + try { + runtime.accept({ kind: "execution" }); + assertEquals( + await f.peer!.request(executorStateOperations.refreshProjectSteering, { + capabilityId: "steering", + availableToolNames: [selectedTool], + }), + "Refreshed", + ); + for (const tool of [rejectedTool, "ungranted_host_tool"]) { + await assertRejects(() => + f.peer!.request(executorStateOperations.refreshProjectSteering, { + capabilityId: "steering", + availableToolNames: [tool], + }) + ); + } + assertEquals(selections, [[selectedTool]]); + } finally { + await runtime.close(); + await broker.shutdown(); + await broker.settled; + } + }); + } + + it("uses owner-scoped tool grants for steering refresh authorization", async () => { + const f = fixture(); + f.input.installation.grant.allowedToolNames = ["fetch-paper"]; + f.input.tools.catalog = new Map([ + ["fetch-paper", {}], + ["coder--fetch-paper", { ownerAgentId: "coder", shortName: "fetch-paper" }], + ]); + f.input.installation.capabilities.projectSteering = "steering"; + f.input.state.prepareProjectSteering = ({ definition }) => + Promise.resolve({ agent: definition }); + let selection: readonly string[] | undefined; + f.input.state.refreshProjectSteering = (_signal, names) => { + selection = names; + return Promise.resolve("Refreshed"); + }; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const runtime = await broker.start(f.input); + try { + runtime.accept({ kind: "execution" }); + assertEquals( + await f.peer!.request(executorStateOperations.refreshProjectSteering, { + capabilityId: "steering", + availableToolNames: ["coder--fetch-paper"], + }), + "Refreshed", + ); + assertEquals(selection, ["coder--fetch-paper"]); + assertEquals(f.installed!.grant.allowedToolNames, ["coder--fetch-paper"]); + await assertRejects(() => + f.peer!.request(executorStateOperations.refreshProjectSteering, { + capabilityId: "steering", + availableToolNames: ["fetch-paper"], + }) + ); + await assertRejects(() => + f.peer!.request(executorStateOperations.refreshProjectSteering, { + capabilityId: "steering", + availableToolNames: ["writer--fetch-paper"], + }) + ); + assertEquals(selection, ["coder--fetch-paper"]); + } finally { + await runtime.close(); + await broker.shutdown(); + await broker.settled; + } + }); + + it("installs, discovers, prepares, accepts, and begins execution in exact order", async () => { + const f = fixture({ initialCheckpoint: true }); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + let admitted = false; + const pending = broker.start(f.input, { + onAdmitted(settled) { + admitted = true; + assert(settled instanceof Promise); + }, + }); + assertEquals(admitted, true); + f.input.model.grant.models.get(modelId)!.maxOutputTokens = 1; + f.input.persistence.initialToolExposureCheckpoint!.loadedToolNames[0] = "changed"; + const runtime = await pending; + assertEquals(f.calls.slice(0, 5), ["allocate", "connect", "install", "describe", "prepare"]); + assertEquals(f.preparationDenied, true); + assertEquals(f.initialCheckpointRead, true); + assertEquals(runtime.definition.id, "coder"); + await assertRejects(() => + runtime.agent.stream({ messages: [], abortSignal: new AbortController().signal }) + ); + runtime.accept({ kind: "execution" }); + await runtime.agent.stream({ messages: [], abortSignal: new AbortController().signal }); + assertEquals(f.executionAllowed, true); + await runtime.close("completed"); + await runtime.settled; + await f.peer?.closed; + await broker.shutdown(); + await broker.settled; + assertEquals(broker.active, 0); + }); + + it("rejects canonical model dispatch without a real event sink before allocation", async () => { + const f = fixture(); + f.input.installation.grant.execution = { + kind: "canonical", + projectId: null, + conversationId: "conversation-1", + runId: "run-1", + messageId: "message-1", + providerReplay: "disabled", + }; + f.input.installation.capabilities.persistence = { + publishParentRunEvents: "parent-events", + toolExposureCheckpoint: "tool-checkpoint", + }; + f.input.persistence = { + publishParentRunEvents: () => Promise.resolve(), + persistToolExposureCheckpoint: () => Promise.resolve(), + }; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + await assertRejects(() => broker.start(f.input)); + assertEquals(f.calls, []); + assertEquals(broker.active, 0); + await broker.shutdown(); + + const ephemeral = fixture(); + ephemeral.input.model.runEventSink = () => Promise.resolve(); + const ephemeralBroker = createManagedExecutorBroker({ maxActive: 1 }); + await assertRejects(() => ephemeralBroker.start(ephemeral.input)); + assertEquals(ephemeral.calls, []); + await ephemeralBroker.shutdown(); + }); + + it("requires canonical session-work binding before allocation", async () => { + const f = fixture(); + configureCanonical(f.input, () => Promise.resolve()); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + + await assertRejects(() => broker.start(f.input)); + assertEquals(f.calls, []); + assertEquals(broker.active, 0); + await broker.shutdown(); + }); + + it("binds canonical persistence ownership immediately after pool admission", async () => { + const f = fixture(); + configureCanonical(f.input, () => Promise.resolve(), (owner) => { + f.calls.push("bind-owned-work"); + assertEquals(typeof owner, "function"); + }); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const runtime = await broker.start(f.input); + + assertEquals(f.calls.slice(0, 3), ["allocate", "bind-owned-work", "connect"]); + await runtime.close(); + await runtime.settled; + await broker.shutdown(); + }); + + it("closes and releases a session when remote preparation fails", async () => { + const f = fixture({ prepareFailure: true }); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + await assertRejects(() => broker.start(f.input), Error, "EXECUTOR_RUNTIME_PREPARATION_FAILED"); + assert(f.calls.includes("release:canceled")); + assertEquals(broker.active, 0); + await f.peer?.closed; + await broker.shutdown(); + await broker.settled; + }); + + it("rejects a prepared model outside the broker selection", async () => { + const f = fixture({ prepareModelId: "veryfront-cloud/openai/other" }); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + await assertRejects(() => broker.start(f.input), Error, "EXECUTOR_RUNTIME_NOT_GRANTED"); + assert(f.calls.includes("release:canceled")); + await broker.shutdown(); + await broker.settled; + }); + + it("returns preparation failure after bounded close while retaining noncooperative work", async () => { + const f = fixture({ brokerReadWait: true }); + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const pending = broker.start(f.input); + await f.prepareEntered; + f.preparation.abort(); + await assertRejects(() => pending); + assertEquals(broker.active, 1); + const shutdown = broker.shutdown(); + f.releasePrepare(); + await shutdown; + await broker.settled; + assertEquals(broker.active, 0); + }); + + it("preserves request cancellation and transfers durable cancellation at acceptance", async () => { + const requestFixture = fixture(); + const requestBroker = createManagedExecutorBroker({ maxActive: 1 }); + const requestRuntime = await requestBroker.start(requestFixture.input); + requestRuntime.accept({ kind: "request" }); + requestFixture.preparation.abort(); + await requestRuntime.settled; + assert(requestFixture.calls.includes("release:canceled")); + await requestBroker.shutdown(); + + const durableFixture = fixture(); + const durableBroker = createManagedExecutorBroker({ maxActive: 1 }); + const durableRuntime = await durableBroker.start(durableFixture.input); + const execution = new AbortController(); + durableRuntime.accept({ kind: "execution", signal: execution.signal }); + durableFixture.preparation.abort(); + await tick(); + assertEquals(durableBroker.active, 1); + execution.abort(); + await durableRuntime.settled; + assert(durableFixture.calls.includes("release:canceled")); + await durableBroker.shutdown(); + }); + + it("retains pool admission until cancelled durable model persistence actually settles", async () => { + const f = fixture(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + f.input.installation.grant.execution = { + kind: "canonical", + projectId: null, + conversationId: "conversation-1", + runId: "run-1", + messageId: "message-1", + providerReplay: "disabled", + }; + f.input.installation.capabilities.persistence = { + publishParentRunEvents: "parent-events", + toolExposureCheckpoint: "tool-checkpoint", + }; + f.input.persistence = { + publishParentRunEvents: () => Promise.resolve(), + persistToolExposureCheckpoint: () => Promise.resolve(), + }; + f.input.model.runEventSink = async () => { + entered.resolve(); + await release.promise; + }; + f.input.bindSessionOwnedWork = () => {}; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const runtime = await broker.start(f.input); + runtime.accept({ kind: "execution" }); + const opening = runtime.agent.stream({ + messages: [], + abortSignal: new AbortController().signal, + }); + void opening.catch(() => {}); + await entered.promise; + await runtime.close(); + let settled = false; + void runtime.settled.then(() => settled = true); + await tick(); + assertEquals(settled, false); + assertEquals(broker.active, 1); + release.resolve(); + await assertRejects(() => opening); + await runtime.settled; + assertEquals(broker.active, 0); + await f.peer?.closed; + await broker.shutdown(); + }); + + it("retains max-active admission for the original canonical append after its deadline", async () => { + using time = new FakeTime(); + const f = fixture({ allocationLifetimeMs: 90_000, hardDeadlineMs: 120_000 }); + const conversationId = "00000000-0000-4000-8000-000000000001"; + const messageId = "00000000-0000-4000-8000-000000000002"; + const appendEntered = Promise.withResolvers(); + const appendRelease = Promise.withResolvers(); + const terminalCalls: Record[] = []; + let delayAuditAppend = true; + const fetch = async (_input: RequestInfo | URL, init?: RequestInit) => { + const body = init?.body ? JSON.parse(String(init.body)) as Record : {}; + if (delayAuditAppend && Array.isArray(body.events)) { + delayAuditAppend = false; + appendEntered.resolve(); + return await appendRelease.promise; + } + terminalCalls.push(body); + return Response.json({ + completed: true, + run: { runId: "run-1", status: body.status }, + }); + }; + const persistence = createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run: { + runId: "run-1", + conversationId, + messageId, + latestEventId: 0, + latestExternalEventSequence: 0, + waitingToolCallId: null, + waitingToolName: null, + status: "running", + streamProtocolVersion: 2, + }, + modelId, + resolveProvider: () => "provider", + fetch, + }); + configureCanonical( + f.input, + persistence.modelRunEventSink, + persistence.bindSessionOwnedWork, + ); + f.input.installation.grant.execution = { + kind: "canonical", + projectId: null, + conversationId, + runId: "run-1", + messageId, + providerReplay: "disabled", + }; + f.input.persistence = { + publishParentRunEvents: persistence.publishParentRunEvents, + persistToolExposureCheckpoint: persistence.persistToolExposureCheckpoint, + }; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + const runtime = await broker.start(f.input); + runtime.accept({ kind: "execution" }); + const opening = runtime.agent.stream({ + messages: [], + abortSignal: new AbortController().signal, + }); + await appendEntered.promise; + + await time.tickAsync(30_000); + const openingResult = await Promise.allSettled([opening]); + const streamError = openingResult[0]?.status === "rejected" + ? openingResult[0].reason + : undefined; + assert(streamError instanceof Error); + + const finishResult = await Promise.allSettled([ + runtime.runOwned(() => persistence.output.finish({ completed: false, error: streamError })), + ]); + assertEquals(finishResult[0]?.status, "rejected"); + assertEquals( + finishResult[0]?.status === "rejected" && finishResult[0].reason instanceof Error + ? finishResult[0].reason.message + : undefined, + "Durable run event persistence timed out", + ); + assertEquals(terminalCalls.length, 1); + assertEquals(terminalCalls.at(-1)?.status, "failed"); + + const closing = runtime.close(); + await time.tickAsync(50); + await closing; + let settled = false; + void runtime.settled.then(() => settled = true); + for (let index = 0; index < 20; index += 1) await Promise.resolve(); + assertEquals(settled, false); + assertEquals(broker.active, 1); + + const second = fixture(); + await assertRejects(() => broker.start(second.input)); + appendRelease.resolve(Response.json({ + latest_event_id: 1, + latest_external_event_sequence: 1, + appended_count: 1, + run: { + run_id: "run-1", + conversation_id: conversationId, + latest_event_id: 1, + latest_external_event_sequence: 1, + }, + })); + await runtime.settled; + await persistence.cleanup(); + assertEquals(broker.active, 0); + + const nextRuntime = await broker.start(second.input); + await nextRuntime.close(); + await nextRuntime.settled; + await broker.shutdown(); + }); + + it("exports a strict preparation result schema", () => { + assertEquals( + getExecutorRuntimePrepareResultSchema().safeParse({ + ok: true, + value: { preparedRuntimeHandle: "handle", runtimeKind: "framework", modelId }, + }).success, + true, + ); + assertEquals( + getExecutorRuntimePrepareResultSchema().safeParse({ + ok: false, + code: "EXECUTOR_RUNTIME_PREPARATION_FAILED", + }).success, + true, + ); + assertEquals( + getExecutorRuntimePrepareResultSchema().safeParse({ + ok: true, + value: { + preparedRuntimeHandle: "handle", + runtimeKind: "framework", + modelId, + token: "secret", + }, + }).success, + false, + ); + }); +}); diff --git a/src/agent/hosted/managed-executor-broker.ts b/src/agent/hosted/managed-executor-broker.ts new file mode 100644 index 0000000000..68c6e07c87 --- /dev/null +++ b/src/agent/hosted/managed-executor-broker.ts @@ -0,0 +1,535 @@ +import type { AgentRunEventSink } from "#veryfront/runtime/model-call-context.ts"; +import type { RuntimeAgentMarkdownDefinition } from "../runtime/agent-definition.ts"; +import type { AgentModelRuntimeResolver } from "../runtime/model-transport.ts"; +import { + createExecutorOperationGate, + type ExecutorOperationGate, +} from "../executor/operation-gate.ts"; +import type { ExecutorBinding } from "../executor/protocol.ts"; +import type { HostedChatRuntimeAgent } from "./chat-runtime-contract.ts"; +import { + createHostedExecutorSessionPool, + type HostedExecutorSessionPoolOptions, +} from "./executor-session-pool.ts"; +import type { + HostedExecutorOwnedWork, + HostedExecutorSessionCloseResult, + HostedExecutorSessionOptions, +} from "./executor-session.ts"; +import { sameHostedExecutorOwner } from "./executor-session-schema.ts"; +import { verifyHostedRuntimeSourceBinding } from "./runtime-source-binding.ts"; +import { + type ExecutorRuntimeInstall, + getExecutorRuntimeInstallSchema, + parseExecutorInstallation, +} from "./executor-runtime-install-schema.ts"; +import { + ExecutorRuntimePreparationError, + type ExecutorRuntimePrepareRequest, + getExecutorRuntimePrepareRequestSchema, + getExecutorRuntimePrepareResultSchema, + isExecutorRuntimePreparationFailureCode, + parseRuntimePreparationData, +} from "./executor-runtime-prepare-schema.ts"; +import { + ExecutorDiscoveryError, + getExecutorAgentDescribeResultSchema, + parseDiscoveryData, +} from "./executor-discovery-schema.ts"; +import { ExecutorAgentError } from "./executor-agent-schema.ts"; +import { createExecutorHostedChatRuntimeAgent } from "./executor-agent-bridge.ts"; +import { + createEphemeralHostedExecutorModelBroker, + createHostedExecutorModelBroker, +} from "./executor-model-dispatch.ts"; +import type { ExecutorModelGrant } from "./executor-model-grant.ts"; +import { createExecutorToolBroker, type ExecutorToolCapability } from "./executor-tool-bridge.ts"; +import { createExecutorPersistenceBroker } from "./executor-persistence-bridge.ts"; +import { executorInitialCheckpointsOperation } from "./executor-checkpoint-state.ts"; +import { createExecutorStateBroker } from "./executor-state-bridge.ts"; +import { executorStateOperations } from "./executor-state-schema.ts"; +import type { ExecutorOperation } from "../executor/channel.ts"; + +type SessionInput = Omit; +type InstallInput = Omit; +type PersistenceInput = Omit< + Parameters[0], + "expectedBinding" | "capabilityIds" +>; +type StateInput = Omit< + Parameters[0], + "expectedBinding" | "capabilityIds" | "agentId" | "projectId" | "branchId" | "allowedToolNames" +>; + +/** Prepared executor handle with broker-owned execution and retirement. */ +export interface ManagedExecutorRuntime { + readonly definition: RuntimeAgentMarkdownDefinition; + readonly modelId: string; + readonly runtimeKind: "framework"; + readonly agent: HostedChatRuntimeAgent; + readonly settled: Promise; + readonly accepted: boolean; + /** Broker-only work; must not await this runtime's close or settled promise. */ + runOwned(operation: () => Promise): Promise; + accept(ownership: { kind: "request" } | { kind: "execution"; signal?: AbortSignal }): void; + close(reason?: "completed" | "canceled"): Promise; +} + +/** + * Trusted per-invocation source, model, tool, persistence, and state authority. + * Broker model limits, provider tools, and tool capabilities must not exceed + * the corresponding installed grant. Startup rejects mismatches before allocation. + */ +export interface ManagedExecutorStartInput { + /** Bind canonical persistence to the admitted session before readiness work starts. */ + bindSessionOwnedWork?: (owner: HostedExecutorOwnedWork) => void; + session: SessionInput; + installation: InstallInput; + prepare: ExecutorRuntimePrepareRequest; + model: { + resolver: AgentModelRuntimeResolver; + grant: ExecutorModelGrant; + runEventSink?: AgentRunEventSink; + }; + tools: { + /** Complete trusted inventory, including project-local tools, before selector resolution. */ + catalog: ReadonlyMap; + sources: ReadonlyMap; + maxCalls: number; + maxConcurrent: number; + limits?: Parameters[0]["limits"]; + }; + persistence: PersistenceInput; + state: StateInput; +} +type ManagedExecutorOperationInput = Pick< + ManagedExecutorStartInput, + "model" | "tools" | "persistence" | "state" +>; + +/** Process admission and shutdown limits for a managed broker. */ +export type ManagedExecutorBrokerOptions = Omit< + HostedExecutorSessionPoolOptions, + "createSession" +>; + +/** Compose an executor pool with authenticated installation and operation gates. */ +export function createManagedExecutorBroker(options: ManagedExecutorBrokerOptions) { + const pool = createHostedExecutorSessionPool(options); + + async function start( + input: ManagedExecutorStartInput, + lifecycle: { onAdmitted?(settled: Promise): void } = {}, + ): Promise { + // Generation one is used only to validate local operation descriptors. + // Actual authority is rebuilt against the allocator's authenticated binding. + const validationBinding: ExecutorBinding = { + allocationId: input.session.request.allocationId, + generation: 1, + invocationId: input.session.request.invocationId, + }; + let installation = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), { + ...input.installation, + binding: validationBinding, + }); + const prepare = parseRuntimePreparationData( + getExecutorRuntimePrepareRequestSchema(), + input.prepare, + ); + const selectedModelId = prepare.modelId ?? installation.grant.defaultModelId; + if ( + !sameHostedExecutorOwner(installation.owner, input.session.request.owner) || + verifyHostedRuntimeSourceBinding(input.session.request.source, installation.source) !== + undefined || + prepare.agentId !== installation.grant.agentId + ) throw new TypeError("Managed executor installation does not match its session"); + const allowedModelIds = new Set(installation.grant.models.map((model) => model.id)); + const operationInput = snapshotOperationInput(input); + constrainInstalledOperationGrants(operationInput, installation); + installation = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), installation); + const bindSessionOwnedWork = input.bindSessionOwnedWork; + if ( + installation.grant.execution.kind === "ephemeral" && + operationInput.model.runEventSink !== undefined + ) { + throw new TypeError("Ephemeral executor model dispatch cannot receive a run event sink"); + } + if ( + installation.grant.execution.kind === "canonical" && + typeof bindSessionOwnedWork !== "function" + ) { + throw new TypeError("Canonical executor persistence requires session-owned work binding"); + } + if (bindSessionOwnedWork !== undefined && typeof bindSessionOwnedWork !== "function") { + throw new TypeError("Invalid executor session-owned work binder"); + } + // Validate every trusted capability before reserving pool admission. + buildBrokerOperations( + validationBinding, + new AbortController().signal, + operationInput, + installation, + allowedModelIds, + selectedModelId, + ); + + let gate: ExecutorOperationGate | undefined; + const session = pool.start({ + ...input.session, + createOperations(binding, signal) { + const channelBinding = toChannelBinding(binding); + const operations = buildBrokerOperations( + channelBinding, + signal, + operationInput, + installation, + allowedModelIds, + selectedModelId, + ); + gate = createExecutorOperationGate({ + binding: channelBinding, + signal, + operations, + preparationOperations: new Set([ + ...Object.values(executorStateOperations), + executorInitialCheckpointsOperation, + ].filter((name) => operations.has(name))), + }); + return { operations: gate.operations, revoke: gate.revoke }; + }, + }); + try { + bindSessionOwnedWork?.(session.runOwned.bind(session)); + lifecycle.onAdmitted?.(session.settled); + const channel = await session.ready; + const binding = session.binding; + if (!binding || !gate) throw new Error("Managed executor session is not bound"); + const channelBinding = toChannelBinding(binding); + const installRequest = parseExecutorInstallation(getExecutorRuntimeInstallSchema(), { + ...installation, + binding: channelBinding, + }); + const installed = await channel.request("runtime.install", installRequest); + if ( + !installed || typeof installed !== "object" || Array.isArray(installed) || + Object.keys(installed).length !== 1 || installed.installed !== true + ) throw new Error("Managed executor installation acknowledgement is invalid"); + const description = parseDiscoveryData( + getExecutorAgentDescribeResultSchema(), + await channel.request("agent.describe", { agentId: prepare.agentId }), + true, + ); + if (!description.ok) throw new ExecutorDiscoveryError(description.code); + if ( + description.value.definition.id !== prepare.agentId || + verifyHostedRuntimeSourceBinding(installation.source, description.value.source) !== + undefined + ) throw new ExecutorDiscoveryError("EXECUTOR_DISCOVERY_INVALID_OUTPUT"); + const prepared = parseRuntimePreparationData( + getExecutorRuntimePrepareResultSchema(), + await channel.request("runtime.prepare", prepare), + ); + if (!prepared.ok) { + if (isExecutorRuntimePreparationFailureCode(prepared.code)) { + throw new ExecutorRuntimePreparationError(prepared.code); + } + throw new ExecutorAgentError(prepared.code); + } + if ( + !allowedModelIds.has(prepared.value.modelId) || prepared.value.modelId !== selectedModelId + ) { + throw new ExecutorRuntimePreparationError("EXECUTOR_RUNTIME_NOT_GRANTED"); + } + gate.markPrepared(); + const remoteAgent = createExecutorHostedChatRuntimeAgent({ + channel, + preparedRuntimeHandle: prepared.value.preparedRuntimeHandle, + }); + const agent: HostedChatRuntimeAgent = { + async stream(streamInput) { + if (!session.accepted || gate!.state !== "prepared") { + throw new Error("Managed executor runtime is not accepted"); + } + gate!.beginExecution(); + return await remoteAgent.stream(streamInput); + }, + }; + const settled = Promise.all([session.settled, gate.settled]).then(() => undefined); + return { + definition: description.value.definition, + modelId: prepared.value.modelId, + runtimeKind: prepared.value.runtimeKind, + agent, + settled, + runOwned: session.runOwned.bind(session), + get accepted() { + return session.accepted; + }, + accept(ownership) { + session.accept(ownership); + }, + close(reason = "canceled") { + gate!.revoke(); + return session.close(reason); + }, + } as ManagedExecutorRuntime; + } catch (error) { + gate?.revoke(); + await session.close("canceled").catch(() => {}); + // The pool retains admission until raw session/gate work settles. Startup + // failure returns after bounded close notification, even for noncooperative work. + void Promise.allSettled([session.settled, gate?.settled]); + throw error; + } + } + + return { + get active() { + return pool.active; + }, + signal: pool.signal, + closed: pool.closed, + settled: pool.settled, + start, + shutdown: pool.shutdown.bind(pool), + }; +} + +function constrainInstalledOperationGrants( + input: ManagedExecutorOperationInput, + installation: ExecutorRuntimeInstall, +): void { + for (const installed of installation.grant.models) { + const policy = input.model.grant.models.get(installed.id); + // The model broker validates missing policies, IDs, and malformed limits. + if (!policy) continue; + if (policy.maxOutputTokens > installed.maxOutputTokens) { + throw new TypeError("Broker model output allowance exceeds the installed model grant"); + } + const allowedProviderTools = new Set(installed.providerToolNames); + if (policy.providerTools.some((tool) => !allowedProviderTools.has(tool.name))) { + throw new TypeError("Broker provider tool policy exceeds the installed model grant"); + } + // Preparation must produce requests that fit the broker's effective policy. + installed.maxOutputTokens = policy.maxOutputTokens; + installed.providerToolNames = policy.providerTools.map((tool) => tool.name); + } + const allowedTools = installedToolNames(installation, input.tools.catalog); + const allowedSources = new Set([ + ...installation.grant.hostToolFacadeIds, + ...installation.grant.remoteToolSourceIds, + ]); + for (const [sourceId, capability] of input.tools.sources) { + if (!allowedSources.has(sourceId)) { + throw new TypeError("Broker tool source exceeds the installed source grant"); + } + for (const name of capability.allowedToolNames) { + if (!allowedTools.has(name)) { + throw new TypeError("Broker tool capability exceeds the installed tool grant"); + } + } + } + installation.grant.allowedToolNames = [...allowedTools]; + // Source listings describe callable tools; only trusted ingress supplies ownership. + const hostToolAliases: NonNullable = []; + for (const sourceId of installation.grant.hostToolFacadeIds) { + for (const toolName of input.tools.sources.get(sourceId)?.allowedToolNames ?? []) { + const metadata = input.tools.catalog.get(toolName); + if ( + metadata?.ownerAgentId === installation.grant.agentId && metadata.shortName !== undefined + ) { + hostToolAliases.push({ + sourceId, + toolName, + ownerAgentId: metadata.ownerAgentId, + shortName: metadata.shortName, + }); + } + } + } + if (hostToolAliases.length) installation.hostToolAliases = hostToolAliases; + else delete installation.hostToolAliases; +} + +function installedToolNames( + installation: ExecutorRuntimeInstall, + catalog: ManagedExecutorStartInput["tools"]["catalog"], +): Set { + const allowed = new Set(); + const agentId = installation.grant.agentId; + for (const selector of installation.grant.allowedToolNames) { + let owned: string | undefined; + for (const [id, tool] of catalog) { + if (tool.ownerAgentId !== agentId || tool.shortName !== selector) continue; + if (owned !== undefined && owned !== id) { + throw new TypeError("Managed executor tool catalog has an ambiguous owned selector"); + } + owned = id; + } + const resolved = owned ?? selector; + const tool = catalog.get(resolved); + if (tool && (tool.ownerAgentId === undefined || tool.ownerAgentId === agentId)) { + allowed.add(resolved); + } + } + return allowed; +} + +function buildBrokerOperations( + binding: ExecutorBinding, + signal: AbortSignal, + input: ManagedExecutorOperationInput, + installation: ExecutorRuntimeInstall, + allowedModelIds: ReadonlySet, + selectedModelId: string, +): ReadonlyMap { + const scope = { binding, signal, assertActive: () => signal.throwIfAborted() }; + const model = installation.grant.execution.kind === "canonical" + ? createHostedExecutorModelBroker({ + resolveModelRuntime: input.model.resolver, + allowedModelIds, + scope, + grant: input.model.grant, + runEventSink: input.model.runEventSink, + }) + : createEphemeralHostedExecutorModelBroker({ + resolveModelRuntime: input.model.resolver, + allowedModelIds, + scope, + grant: input.model.grant, + prepared: { conversationId: null, canonicalRootRun: null }, + }); + const tools = createExecutorToolBroker({ scope, ...input.tools }); + const persistence = createExecutorPersistenceBroker({ + expectedBinding: binding, + capabilityIds: installation.capabilities.persistence, + ...input.persistence, + }); + const execution = installation.grant.execution; + const state = createExecutorStateBroker({ + expectedBinding: binding, + capabilityIds: { + projectSteering: installation.capabilities.projectSteering, + conversationUserText: installation.capabilities.conversationUserText, + }, + agentId: installation.grant.agentId, + projectId: execution.projectId, + branchId: execution.branchId, + ...input.state, + allowedToolNames: [ + ...installation.grant.allowedToolNames, + ...(installation.grant.models.find((model) => model.id === selectedModelId) + ?.providerToolNames ?? + []), + ], + }); + const combined = new Map(); + for (const operations of [model, tools, persistence, state]) { + for (const [name, operation] of operations) { + if (combined.has(name)) throw new TypeError("Managed executor operation collision"); + combined.set(name, operation); + } + } + return combined; +} + +function snapshotOperationInput(input: ManagedExecutorStartInput): ManagedExecutorOperationInput { + if (input.tools.catalog === undefined) { + throw new TypeError("Managed executor tool catalog is required"); + } + const catalog = new Map([...input.tools.catalog].map(([id, tool]) => [ + id, + Object.freeze({ + ownerAgentId: tool.ownerAgentId, + shortName: tool.shortName, + }), + ])); + const sources = new Map(); + for (const [id, capability] of input.tools.sources) { + const source = capability.source; + const context = capability.context; + sources.set(id, { + source: Object.freeze({ + id: source.id, + listTools: source.listTools.bind(source), + executeTool: source.executeTool.bind(source), + }), + allowedToolNames: new Set(capability.allowedToolNames), + context: Object.freeze({ + ...context, + ...(context.publishDataEvent + ? { publishDataEvent: context.publishDataEvent.bind(context) } + : {}), + }), + }); + } + return { + model: { + resolver: input.model.resolver, + grant: { + maxCalls: input.model.grant.maxCalls, + maxConcurrentCalls: input.model.grant.maxConcurrentCalls, + models: new Map([...input.model.grant.models].map(([id, policy]) => [id, { + maxOutputTokens: policy.maxOutputTokens, + providerTools: structuredClone(policy.providerTools), + }])), + }, + ...(input.model.runEventSink ? { runEventSink: input.model.runEventSink } : {}), + }, + tools: { + catalog, + sources, + maxCalls: input.tools.maxCalls, + maxConcurrent: input.tools.maxConcurrent, + ...(input.tools.limits ? { limits: { ...input.tools.limits } } : {}), + }, + persistence: { + ...(input.persistence.initialToolExposureCheckpoint + ? { + initialToolExposureCheckpoint: structuredClone( + input.persistence.initialToolExposureCheckpoint, + ), + } + : {}), + ...(input.persistence.initialProviderReplayCheckpoints + ? { + initialProviderReplayCheckpoints: structuredClone( + input.persistence.initialProviderReplayCheckpoints, + ), + } + : {}), + ...(input.persistence.publishParentRunEvents + ? { publishParentRunEvents: input.persistence.publishParentRunEvents } + : {}), + ...(input.persistence.persistToolExposureCheckpoint + ? { persistToolExposureCheckpoint: input.persistence.persistToolExposureCheckpoint } + : {}), + ...(input.persistence.persistProviderReplayCheckpoint + ? { persistProviderReplayCheckpoint: input.persistence.persistProviderReplayCheckpoint } + : {}), + }, + state: { + ...(input.state.prepareProjectSteering + ? { prepareProjectSteering: input.state.prepareProjectSteering } + : {}), + ...(input.state.refreshProjectSteering + ? { refreshProjectSteering: input.state.refreshProjectSteering } + : {}), + ...(input.state.latestConversationUserText + ? { latestConversationUserText: input.state.latestConversationUserText } + : {}), + }, + }; +} + +function toChannelBinding(binding: { + allocationId: string; + generation: number; + invocationId: string; +}): ExecutorBinding { + return { + allocationId: binding.allocationId, + generation: binding.generation, + invocationId: binding.invocationId, + }; +} diff --git a/src/agent/runtime/project-files-client.test.ts b/src/agent/runtime/project-files-client.test.ts index 69a8a83841..3369b99f2c 100644 --- a/src/agent/runtime/project-files-client.test.ts +++ b/src/agent/runtime/project-files-client.test.ts @@ -1047,6 +1047,7 @@ Deno.test("strict runtime project files preserve timeout identity across request Deno.test("strict project file requests enforce monotonic timeouts around synchronous fetch work", async () => { const responseCancelled = createDeferred(); + let fetchCalls = 0; const response = streamResponse( [new TextEncoder().encode('{"path":"src/index.ts","content":"ok"}')], {}, @@ -1055,9 +1056,10 @@ Deno.test("strict project file requests enforce monotonic timeouts around synchr const error = await assertRejects(() => getStrictRuntimeProjectFile({ ...baseOptions, - timeoutMs: 1, + timeoutMs: TEST_IN_FLIGHT_DEADLINE_MS, fetch: async () => { - const busyUntil = performance.now() + 20; + fetchCalls += 1; + const busyUntil = performance.now() + TEST_IN_FLIGHT_DEADLINE_MS + 20; while (performance.now() < busyUntil) { // Deliberately block timer delivery to verify the monotonic check. } @@ -1069,6 +1071,7 @@ Deno.test("strict project file requests enforce monotonic timeouts around synchr assertEquals((error as Error).name, "TimeoutError"); assertStringIncludes(getErrorMessage(error), "request timed out"); + assertEquals(fetchCalls, 1, "the deadline must expire after synchronous fetch work starts"); await responseCancelled.promise; }); diff --git a/src/agent/service/broker-credentials.test.ts b/src/agent/service/broker-credentials.test.ts new file mode 100644 index 0000000000..1303db3dfb --- /dev/null +++ b/src/agent/service/broker-credentials.test.ts @@ -0,0 +1,34 @@ +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { containsBrokerCredential } from "./broker-credentials.ts"; + +describe("broker credential text normalization", () => { + for ( + const [name, value] of [ + ["malformed escape prefix", "%ZZ api%2Dauth%2Dtoken"], + ["invalid UTF-8 prefix", "%FF%61%70%69%2Dauth%2Dtoken"], + ["two URI encoding layers", "api%252Dauth%252Dtoken"], + ["encoded percent and hex characters", "%25%36%31pi%252Dauth%252Dtoken"], + ["encoded Unicode after invalid UTF-8", "%FF%C3%A9-token"], + ] as const + ) { + it(`finds a credential after ${name}`, () => { + assertEquals(containsBrokerCredential(value, ["api-auth-token", "é-token"]), true); + assertEquals( + containsBrokerCredential({ [value]: "value" }, ["api-auth-token", "é-token"]), + true, + ); + }); + } + + it("fails closed when URI nesting exceeds the bounded normalization limit", () => { + const value = "api%" + "25".repeat(32) + "2Dauth-token"; + assertEquals(containsBrokerCredential(value, ["api-auth-token"]), true); + }); + + it("preserves unrelated text and malformed escapes without credentials", () => { + for (const value of ["percent %ZZ text", "%FFhello%20world", "hello%2520world"]) { + assertEquals(containsBrokerCredential(value, ["api-auth-token"]), false); + } + }); +}); diff --git a/src/agent/service/broker-credentials.ts b/src/agent/service/broker-credentials.ts new file mode 100644 index 0000000000..e9bb4e5d91 --- /dev/null +++ b/src/agent/service/broker-credentials.ts @@ -0,0 +1,45 @@ +const MAX_URI_DECODE_PASSES = 16; +const uriTextDecoder = new TextDecoder(); + +/** Reject known credentials or text that exceeds the URI normalization limit. */ +export function containsBrokerCredential( + value: unknown, + credentials: readonly (string | null | undefined)[], +): boolean { + for (const credential of credentials) { + if (!credential) continue; + if (containsString(value, credential)) return true; + const bearer = /^Bearer\s+(.+)$/i.exec(credential)?.[1]; + if (bearer && containsString(value, bearer)) return true; + } + return false; +} + +function containsString(value: unknown, expected: string): boolean { + if (typeof value === "string") return containsCredentialText(value, expected); + if (!value || typeof value !== "object") return false; + return Array.isArray(value) + ? value.some((entry) => containsString(entry, expected)) + : Object.entries(value).some(([key, entry]) => + containsCredentialText(key, expected) || containsString(entry, expected) + ); +} + +function containsCredentialText(value: string, expected: string): boolean { + let text = value; + for (let pass = 0; pass < MAX_URI_DECODE_PASSES; pass++) { + if (text.includes(expected)) return true; + if (!/%[0-9a-f]{2}/i.test(text)) return false; + // Decode valid byte runs independently. A malformed escape or invalid + // UTF-8 prefix must not hide a valid credential later in the same string. + text = text.replace(/(?:%[0-9a-f]{2})+/gi, (encoded) => { + const bytes = new Uint8Array(encoded.length / 3); + for (let index = 0; index < bytes.length; index++) { + bytes[index] = Number.parseInt(encoded.slice(index * 3 + 1, index * 3 + 3), 16); + } + return uriTextDecoder.decode(bytes); + }); + } + // Keep normalization work bounded and reject text that needs more decoding. + return text.includes(expected) || /%[0-9a-f]{2}/i.test(text); +} diff --git a/src/agent/service/broker-ingress.test.ts b/src/agent/service/broker-ingress.test.ts new file mode 100644 index 0000000000..5c83eb0c2c --- /dev/null +++ b/src/agent/service/broker-ingress.test.ts @@ -0,0 +1,386 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { createControlPlaneSignature } from "#veryfront/server/handlers/request/internal-agent-run.test-helpers.ts"; +import { BrokerIngressError, parseBrokerRuntimeAgentIngress } from "./broker-ingress.ts"; + +const projectId = "00000000-0000-4000-8000-000000000005"; +const userId = "00000000-0000-4000-8000-000000000006"; +const conversationId = "00000000-0000-4000-8000-000000000001"; +const messageId = "00000000-0000-4000-8000-000000000002"; +const inputAnchorMessageId = "00000000-0000-4000-8000-000000000003"; +const path = "/api/control-plane/runs/run-1/stream"; + +function invocation(overrides: Record = {}) { + return { + run: { + agentServiceId: "service-1", + agentId: "builder", + conversationId, + runId: "run-1", + messageId, + inputAnchorMessageId, + requestedByUserId: userId, + project: { projectId, projectSlug: "demo-project", runtimeTargetKind: "main_branch" }, + }, + messages: [], + tools: [], + context: [], + agentSource: { type: "release", releaseId: "release-1" }, + credentials: { authToken: "api-auth-token", inferenceAuthToken: "inference-token" }, + ...overrides, + }; +} + +async function signedRequest( + bodyValue = invocation(), + overrides: Parameters[1] = {}, +) { + const rawBody = JSON.stringify(bodyValue); + const signature = await createControlPlaneSignature(rawBody, { + audience: "demo-project", + projectId, + requestId: "run-1", + requestPath: path, + ...overrides, + }); + return { + publicKeyPem: signature.publicKeyPem, + request: new Request(`https://broker.test${path}`, { + method: "POST", + headers: { + authorization: "Bearer broker-token", + "content-type": "application/json", + "x-veryfront-control-plane-jws": signature.jws, + "x-veryfront-run-event-token": "run-event-token", + }, + body: rawBody, + }), + }; +} + +function options(publicKeyPem: string) { + return { + publicKeyPem, + audience: "demo-project", + projectId, + expectedRunId: "run-1", + expectedSurface: "studio" as const, + boundSource: { type: "release" as const, releaseId: "release-1" }, + expectedOwner: { scopeKind: "project" as const, projectId }, + authorizeScope: (input: { + authorization: string; + apiAuthToken: string; + runEventToken: string; + }) => { + assertEquals(input.authorization, "Bearer broker-token"); + assertEquals(input.apiAuthToken, "api-auth-token"); + assertEquals(input.runEventToken, "run-event-token"); + return Promise.resolve({ principal: { userId }, runEventWriter: { id: "writer-1" } }); + }, + }; +} + +describe("managed broker ingress", () => { + it("accepts a canonical run ID encoded in the signed request path", async () => { + const encodedPath = "/api/control-plane/runs/%72un%2D1/stream"; + const signed = await signedRequest(invocation(), { requestPath: encodedPath }); + const encoded = new Request(`https://broker.test${encodedPath}`, { + method: "POST", + headers: signed.request.headers, + body: await signed.request.text(), + }); + const result = await parseBrokerRuntimeAgentIngress(encoded, options(signed.publicKeyPem)); + assertEquals(result.executor.run.runId, "run-1"); + }); + it("verifies the exact body and produces disjoint private authority and executor data", async () => { + const signed = await signedRequest(); + const result = await parseBrokerRuntimeAgentIngress( + signed.request, + options(signed.publicKeyPem), + ); + assertEquals(result.privateAuthority.inferenceAuthToken, "inference-token"); + assertEquals(result.privateAuthority.apiAuthToken, "api-auth-token"); + assertEquals(result.privateAuthority.runEventToken, "run-event-token"); + assertEquals(result.privateAuthority.inboundAuthorization, "Bearer broker-token"); + assertEquals(result.executor.run.runId, "run-1"); + assertEquals(result.executor.input.runId, "run-1"); + const visible = JSON.stringify(result.executor); + for ( + const secret of ["broker-token", "api-auth-token", "run-event-token", "inference-token"] + ) { + assertEquals(visible.includes(secret), false); + } + assertEquals(signed.request.bodyUsed, true); + }); + + for ( + const credential of [ + "api-auth-token", + "inference-token", + "run-event-token", + "Bearer broker-token", + "broker-token", + ] + ) { + it(`rejects embedded ${credential} in application messages`, async () => { + const signed = await signedRequest(invocation({ + messages: [{ + id: "message-1", + role: "user", + content: `Use ${credential} for this request`, + }], + })); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(signed.request, options(signed.publicKeyPem)), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + }); + } + + it("rejects credentials inside nested attachment URLs and property names", async () => { + for ( + const forwardedProps of [ + { attachments: [{ url: "https://files.test/document?token=api-auth-token&download=1" }] }, + { attachments: [{ "result-inference-token-metadata": "value" }] }, + ] + ) { + const signed = await signedRequest(invocation({ forwardedProps })); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(signed.request, options(signed.publicKeyPem)), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + } + }); + + for ( + const [placement, forwardedProps] of [ + ["attachment URL", { + attachments: [{ url: "https://files.test/document?token=api%2Dauth%2Dtoken&download=1" }], + }], + ["property name", { attachments: [{ "result-inference%2Dtoken-metadata": "value" }] }], + ["Bearer attachment URL", { + attachments: [{ url: "https://files.test/document?token=Bearer%20broker%2Dtoken" }], + }], + ] as const + ) { + it(`rejects URI-escaped credentials in an ${placement}`, async () => { + const signed = await signedRequest(invocation({ forwardedProps })); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(signed.request, options(signed.publicKeyPem)), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + }); + } + + it("preserves application strings that do not contain a credential", async () => { + const messages = [{ + id: "message-1", + role: "user" as const, + content: "Explain Bearer authentication", + }]; + const forwardedProps = { + attachments: [{ url: "https://files.test/document?name=a%20b&download=1" }, { + url: "https://files.test/invalid%escape", + }], + }; + const signed = await signedRequest(invocation({ messages, forwardedProps })); + const result = await parseBrokerRuntimeAgentIngress( + signed.request, + options(signed.publicKeyPem), + ); + assertEquals(result.executor.input.messages, messages); + assertEquals(result.executor.input.forwardedProps, forwardedProps); + }); + + it("rejects invalid signatures and signed method, path, or run mismatches", async () => { + const signed = await signedRequest(); + const invalid = new Request(signed.request.url, { + method: "POST", + headers: signed.request.headers, + body: `${JSON.stringify(invocation())} `, + }); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(invalid, options(signed.publicKeyPem)), + 401, + "BROKER_INGRESS_AUTH_INVALID", + ); + + const wrongRun = await signedRequest( + invocation({ run: { ...invocation().run, runId: "run-2" } }), + ); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(wrongRun.request, options(wrongRun.publicKeyPem)), + 400, + "CONTROL_PLANE_RUN_ID_MISMATCH", + ); + }); + + it("rejects invalid, oversized, and aborted bodies before authorization", async () => { + let authorizations = 0; + const signed = await signedRequest(); + const invalidBody = "{"; + const invalidSignature = await createControlPlaneSignature(invalidBody, { + audience: "demo-project", + projectId, + requestId: "run-1", + requestPath: path, + }); + const invalid = new Request(`https://broker.test${path}`, { + method: "POST", + headers: { + authorization: "Bearer broker-token", + "x-veryfront-control-plane-jws": invalidSignature.jws, + "x-veryfront-run-event-token": "run-event-token", + }, + body: invalidBody, + }); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(invalid, { + ...options(invalidSignature.publicKeyPem), + authorizeScope: () => { + authorizations++; + return Promise.resolve({}); + }, + }), + 400, + "BROKER_INGRESS_INVALID_BODY", + ); + + const oversized = new Request(`https://broker.test${path}`, { + method: "POST", + headers: signed.request.headers, + body: "x".repeat(1024 * 1024 + 1), + }); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(oversized, { + ...options(signed.publicKeyPem), + authorizeScope: () => { + authorizations++; + return Promise.resolve({}); + }, + }), + 413, + "BROKER_INGRESS_BODY_TOO_LARGE", + ); + + const controller = new AbortController(); + const aborted = new Request( + `https://broker.test${path}`, + { + method: "POST", + headers: signed.request.headers, + body: new ReadableStream(), + signal: controller.signal, + duplex: "half", + } as RequestInit, + ); + const pending = parseBrokerRuntimeAgentIngress(aborted, { + ...options(signed.publicKeyPem), + authorizeScope: () => { + authorizations++; + return Promise.resolve({}); + }, + readTimeoutMs: 1_000, + }); + controller.abort(); + await assertIngressError(() => pending, 499, "BROKER_INGRESS_ABORTED"); + + const timedOut = new Request( + `https://broker.test${path}`, + { + method: "POST", + headers: signed.request.headers, + body: new ReadableStream(), + duplex: "half", + } as RequestInit, + ); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(timedOut, { + ...options(signed.publicKeyPem), + authorizeScope: () => { + authorizations++; + return Promise.resolve({}); + }, + readTimeoutMs: 1, + }), + 408, + "BROKER_INGRESS_TIMEOUT", + ); + assertEquals(authorizations, 0); + }); + + it("fails closed on source, project, owner, and required credential scope", async () => { + const signed = await signedRequest(); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(signed.request.clone(), { + ...options(signed.publicKeyPem), + boundSource: { type: "release", releaseId: "release-2" }, + }), + 409, + "CONTROL_PLANE_AGENT_SOURCE_MISMATCH", + ); + + const wrongProject = await signedRequest( + invocation({ + run: { + ...invocation().run, + project: { + projectId: "00000000-0000-4000-8000-000000000099", + projectSlug: "demo-project", + runtimeTargetKind: "main_branch", + }, + }, + }), + ); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(wrongProject.request, options(wrongProject.publicKeyPem)), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + + const noEventToken = await signedRequest(); + noEventToken.request.headers.delete("x-veryfront-run-event-token"); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(noEventToken.request, options(noEventToken.publicKeyPem)), + 401, + "BROKER_INGRESS_AUTH_REQUIRED", + ); + + const noApiToken = await signedRequest(invocation({ credentials: undefined })); + await assertIngressError( + () => parseBrokerRuntimeAgentIngress(noApiToken.request, options(noApiToken.publicKeyPem)), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + + const denied = await signedRequest(); + await assertIngressError( + () => + parseBrokerRuntimeAgentIngress(denied.request, { + ...options(denied.publicKeyPem), + authorizeScope: () => Promise.resolve(undefined), + }), + 403, + "BROKER_INGRESS_SCOPE_DENIED", + ); + }); +}); + +async function assertIngressError( + operation: () => Promise, + status: number, + errorCode: string, +) { + const error = await assertRejects(operation, BrokerIngressError) as BrokerIngressError; + assertEquals({ status: error.status, errorCode: error.errorCode }, { status, errorCode }); +} diff --git a/src/agent/service/broker-ingress.ts b/src/agent/service/broker-ingress.ts new file mode 100644 index 0000000000..e4e1bf43dc --- /dev/null +++ b/src/agent/service/broker-ingress.ts @@ -0,0 +1,304 @@ +import { parseBrokerSignedRunPath } from "./broker-run-route.ts"; +import { containsBrokerCredential } from "#veryfront/agent/service/broker-credentials.ts"; +import type { ControlPlaneClaims, ControlPlaneSurface } from "#veryfront/channels/control-plane.ts"; +import { + CONTROL_PLANE_JWS_HEADER, + verifyControlPlaneJws, +} from "#veryfront/channels/control-plane.ts"; +import { + buildRuntimeAgentControlPlaneStreamRequestFromInvocation, + getRuntimeAgentRunIdSchema, + type RuntimeAgentRunContext, + type RuntimeAgentRunInvocation, + safeParseRuntimeAgentRunInvocationValue, +} from "#veryfront/agent/runtime/agent-invocation-contract.ts"; +import { + getInternalAgentControlPlaneStreamRequestSchema, + type RuntimeRunAgentInput, + toRuntimeRunAgentInput, +} from "#veryfront/internal-agents/schema.ts"; +import { + isRequestBodyTooLargeError, + readBodyBytesWithLimit, +} from "#veryfront/security/input-validation/limits.ts"; +import { DEFAULT_MAX_BODY_SIZE_BYTES } from "#veryfront/utils/constants/buffers.ts"; +import { snapshotBoundedJsonValue } from "#veryfront/schemas/json-value.ts"; +import { + getHostedExecutorOwnerSchema, + type HostedExecutorOwner, +} from "#veryfront/agent/hosted/executor-session-schema.ts"; +import { + type HostedRuntimeSourceIdentity, + verifyHostedRuntimeSourceBinding, +} from "#veryfront/agent/hosted/runtime-source-binding.ts"; + +const BROKER_INGRESS_MAX_BODY_BYTES = DEFAULT_MAX_BODY_SIZE_BYTES; +const RUN_EVENT_APPEND_TOKEN_HEADER = "x-veryfront-run-event-token"; +const DEFAULT_BODY_READ_TIMEOUT_MS = 30_000; +const MAX_BODY_READ_TIMEOUT_MS = 60_000; +const fatalDecoder = new TextDecoder("utf-8", { fatal: true }); +const forbiddenForwardedAuthorityFields = new Set([ + "authorization", + "authtoken", + "inferenceauthtoken", + "credential", + "credentials", + "runeventtoken", +]); + +/** Fixed, credential-free ingress failure identifiers. */ +export type BrokerIngressErrorCode = + | "BROKER_INGRESS_INVALID_BODY" + | "BROKER_INGRESS_BODY_TOO_LARGE" + | "BROKER_INGRESS_ABORTED" + | "BROKER_INGRESS_TIMEOUT" + | "BROKER_INGRESS_AUTH_REQUIRED" + | "BROKER_INGRESS_AUTH_INVALID" + | "BROKER_INGRESS_SCOPE_DENIED" + | "BROKER_INGRESS_SCOPE_FAILED" + | "BROKER_INGRESS_TARGET_MISMATCH" + | "CONTROL_PLANE_RUN_ID_MISMATCH" + | "CONTROL_PLANE_AGENT_SOURCE_UNBOUND" + | "CONTROL_PLANE_AGENT_SOURCE_UNSUPPORTED" + | "CONTROL_PLANE_AGENT_SOURCE_MISMATCH"; + +/** + * Local HTTP-boundary errors intentionally avoid the application error registry: + * only a fixed code/status is exposed, never body, credential, or verifier diagnostics. + * These errors stay in the broker and are not executor-channel error contracts. + */ +export class BrokerIngressError extends Error { + constructor(readonly status: number, readonly errorCode: BrokerIngressErrorCode) { + super(errorCode); + this.name = "BrokerIngressError"; + } +} + +/** Validated application data that can cross the executor channel. */ +export interface BrokerRuntimeAgentExecutorInput { + readonly owner: HostedExecutorOwner; + readonly run: Omit & { + validatedClaims?: RuntimeAgentRunContext["validatedClaims"]; + }; + readonly taskId?: string; + readonly agentSource: RuntimeAgentRunInvocation["agentSource"]; + readonly agentConfig?: RuntimeAgentRunInvocation["agentConfig"]; + readonly input: RuntimeRunAgentInput; +} + +/** HTTP credentials and verified authority retained exclusively in the broker. */ +export interface BrokerRuntimeAgentPrivateAuthority { + readonly owner: HostedExecutorOwner; + readonly claims: Readonly; + readonly inboundAuthorization: string; + readonly apiAuthToken: string; + readonly runEventToken: string; + readonly inferenceAuthToken?: string; + readonly authorization: TAuthorization; + readonly rawBody: string; +} + +/** Signed identity and credentials supplied to the trusted scope verifier. */ +export interface BrokerIngressScopeInput { + readonly owner: HostedExecutorOwner; + readonly claims: Readonly; + readonly run: RuntimeAgentRunContext; + readonly authorization: string; + readonly apiAuthToken: string; + readonly runEventToken: string; +} + +/** Broker-owned verification policy for one expected run and source. */ +export interface BrokerRuntimeAgentIngressOptions { + publicKeyPem: string; + audience: string; + projectId: string; + expectedRunId: string; + expectedSurface: ControlPlaneSurface; + boundSource: HostedRuntimeSourceIdentity | undefined; + expectedOwner: HostedExecutorOwner; + authorizeScope( + input: BrokerIngressScopeInput, + ): TAuthorization | undefined | Promise; + signal?: AbortSignal; + readTimeoutMs?: number; +} + +/** Separate private authority and executor-safe invocation data. */ +export interface BrokerRuntimeAgentIngress { + privateAuthority: BrokerRuntimeAgentPrivateAuthority; + executor: BrokerRuntimeAgentExecutorInput; +} + +/** Read and verify a signed invocation once before constructing executor-safe data. */ +export async function parseBrokerRuntimeAgentIngress( + request: Request, + options: BrokerRuntimeAgentIngressOptions, +): Promise> { + const expectedRunId = getRuntimeAgentRunIdSchema().parse(options.expectedRunId); + const actualPath = new URL(request.url).pathname; + if (request.method !== "POST" || parseBrokerSignedRunPath(actualPath) !== expectedRunId) { + throw new BrokerIngressError(400, "BROKER_INGRESS_TARGET_MISMATCH"); + } + const inboundAuthorization = request.headers.get("authorization"); + const signature = request.headers.get(CONTROL_PLANE_JWS_HEADER); + const runEventToken = request.headers.get(RUN_EVENT_APPEND_TOKEN_HEADER); + if ( + !inboundAuthorization || inboundAuthorization.length > 16 * 1024 || !signature || + !runEventToken || runEventToken.length > 16 * 1024 + ) { + throw new BrokerIngressError(401, "BROKER_INGRESS_AUTH_REQUIRED"); + } + const ownerResult = getHostedExecutorOwnerSchema().safeParse(options.expectedOwner); + if (!ownerResult.success) throw new TypeError("Invalid broker ingress owner"); + const owner = Object.freeze(ownerResult.data); + if (owner.scopeKind === "project" && owner.projectId !== options.projectId) { + throw new TypeError("Broker ingress owner does not match its project scope"); + } + + const timeoutMs = options.readTimeoutMs ?? DEFAULT_BODY_READ_TIMEOUT_MS; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > MAX_BODY_READ_TIMEOUT_MS) { + throw new TypeError("Invalid broker ingress body timeout"); + } + const timeoutSignal = AbortSignal.timeout(timeoutMs); + const signals = [request.signal, timeoutSignal, ...(options.signal ? [options.signal] : [])]; + const readSignal = AbortSignal.any(signals); + let rawBody: string; + try { + const bytes = await readBodyBytesWithLimit(request, BROKER_INGRESS_MAX_BODY_BYTES, { + signal: readSignal, + }); + rawBody = fatalDecoder.decode(bytes); + } catch (error) { + if (isRequestBodyTooLargeError(error)) { + throw new BrokerIngressError(413, "BROKER_INGRESS_BODY_TOO_LARGE"); + } + if (readSignal.aborted) { + throw new BrokerIngressError( + timeoutSignal.aborted && !request.signal.aborted && !options.signal?.aborted ? 408 : 499, + timeoutSignal.aborted && !request.signal.aborted && !options.signal?.aborted + ? "BROKER_INGRESS_TIMEOUT" + : "BROKER_INGRESS_ABORTED", + ); + } + throw new BrokerIngressError(400, "BROKER_INGRESS_INVALID_BODY"); + } + + let claims: ControlPlaneClaims; + try { + claims = await verifyControlPlaneJws(signature, rawBody, { + audience: options.audience, + expectedProjectId: options.projectId, + expectedSubject: expectedRunId, + expectedSurface: options.expectedSurface, + maxAgeSeconds: 60, + publicKeyPem: options.publicKeyPem, + requestMethod: request.method, + requestPath: actualPath, + }); + } catch { + throw new BrokerIngressError(401, "BROKER_INGRESS_AUTH_INVALID"); + } + + let bodyValue: unknown; + try { + bodyValue = JSON.parse(rawBody); + } catch { + throw new BrokerIngressError(400, "BROKER_INGRESS_INVALID_BODY"); + } + const invocationResult = safeParseRuntimeAgentRunInvocationValue(bodyValue); + if (!invocationResult.success) { + throw new BrokerIngressError(400, "BROKER_INGRESS_INVALID_BODY"); + } + const invocation = invocationResult.data; + if (invocation.run.runId !== expectedRunId) { + throw new BrokerIngressError(400, "CONTROL_PLANE_RUN_ID_MISMATCH"); + } + if ( + invocation.run.project.projectId !== options.projectId || + invocation.run.project.projectSlug !== options.audience || + claims.project_id !== invocation.run.project.projectId || + claims.aud !== invocation.run.project.projectSlug + ) throw new BrokerIngressError(403, "BROKER_INGRESS_SCOPE_DENIED"); + const sourceError = verifyHostedRuntimeSourceBinding(options.boundSource, invocation.agentSource); + if (sourceError) throw new BrokerIngressError(sourceError.status, sourceError.errorCode); + + const inbound = buildRuntimeAgentControlPlaneStreamRequestFromInvocation(invocation); + const parsedInbound = getInternalAgentControlPlaneStreamRequestSchema().safeParse(inbound); + if (!parsedInbound.success) throw new BrokerIngressError(400, "BROKER_INGRESS_INVALID_BODY"); + const apiAuthToken = invocation.credentials?.authToken; + if (!apiAuthToken) throw new BrokerIngressError(403, "BROKER_INGRESS_SCOPE_DENIED"); + if (containsForwardedAuthority(parsedInbound.data.forwardedProps)) { + throw new BrokerIngressError(403, "BROKER_INGRESS_SCOPE_DENIED"); + } + + let authorization: TAuthorization | undefined; + try { + authorization = await options.authorizeScope({ + owner, + claims: Object.freeze({ ...claims }), + run: invocation.run, + authorization: inboundAuthorization, + apiAuthToken, + runEventToken, + }); + } catch { + throw new BrokerIngressError(500, "BROKER_INGRESS_SCOPE_FAILED"); + } + if (authorization === undefined) { + throw new BrokerIngressError(403, "BROKER_INGRESS_SCOPE_DENIED"); + } + + const executorValue = snapshotExecutorValue( + { + owner, + run: invocation.run, + ...(invocation.taskId ? { taskId: invocation.taskId } : {}), + agentSource: invocation.agentSource, + ...(invocation.agentConfig ? { agentConfig: invocation.agentConfig } : {}), + input: toRuntimeRunAgentInput(parsedInbound.data), + } satisfies BrokerRuntimeAgentExecutorInput, + ); + if ( + containsBrokerCredential(executorValue, [ + inboundAuthorization, + apiAuthToken, + runEventToken, + invocation.credentials?.inferenceAuthToken, + ]) + ) { + throw new BrokerIngressError(403, "BROKER_INGRESS_SCOPE_DENIED"); + } + return { + privateAuthority: Object.freeze({ + owner, + claims: Object.freeze({ ...claims }), + inboundAuthorization, + apiAuthToken, + runEventToken, + ...(invocation.credentials?.inferenceAuthToken + ? { inferenceAuthToken: invocation.credentials.inferenceAuthToken } + : {}), + authorization, + rawBody, + }), + executor: executorValue, + }; +} + +function snapshotExecutorValue(value: T): T { + const snapshot = snapshotBoundedJsonValue(value); + if (!snapshot.success) throw new BrokerIngressError(400, "BROKER_INGRESS_INVALID_BODY"); + // The bounded copy preserves the assembled, schema-validated DTO structure. + return snapshot.value as T; +} + +function containsForwardedAuthority(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(containsForwardedAuthority); + for (const [key, entry] of Object.entries(value)) { + if (forbiddenForwardedAuthorityFields.has(key.replace(/[-_]/g, "").toLowerCase())) return true; + if (containsForwardedAuthority(entry)) return true; + } + return false; +} diff --git a/src/agent/service/broker-run-route.ts b/src/agent/service/broker-run-route.ts new file mode 100644 index 0000000000..c5caa09733 --- /dev/null +++ b/src/agent/service/broker-run-route.ts @@ -0,0 +1,19 @@ +import { getRuntimeAgentRunIdSchema } from "../runtime/agent-invocation-contract.ts"; + +/** Decode one path segment and enforce the canonical runtime run ID contract. */ +export function decodeBrokerRunId(value: string): string | null { + let decoded: string; + try { + decoded = decodeURIComponent(value); + } catch { + return null; + } + const parsed = getRuntimeAgentRunIdSchema().safeParse(decoded); + return parsed.success ? parsed.data : null; +} + +/** Read the signed stream target without changing the path used for signature verification. */ +export function parseBrokerSignedRunPath(pathname: string): string | null { + const match = /^\/api\/control-plane\/runs\/([^/]+)\/stream$/u.exec(pathname); + return match ? decodeBrokerRunId(match[1]!) : null; +} diff --git a/src/agent/service/managed-broker-handler.test.ts b/src/agent/service/managed-broker-handler.test.ts new file mode 100644 index 0000000000..b37b57bd81 --- /dev/null +++ b/src/agent/service/managed-broker-handler.test.ts @@ -0,0 +1,634 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { createControlPlaneSignature } from "#veryfront/server/handlers/request/internal-agent-run.test-helpers.ts"; +import type { + ManagedExecutorRuntime, + ManagedExecutorStartInput, +} from "../hosted/managed-executor-broker.ts"; +import { createManagedBrokerHandler } from "./managed-broker-handler.ts"; +import { ExecutorAgentError } from "../hosted/executor-agent-schema.ts"; +import { resolveConversationHostedStreamErrorState } from "../conversation/hosted-terminal.ts"; +import { agUiSseEventTypes, parseAgUiSseResponse } from "../ag-ui/sse-parser.ts"; + +const projectId = "00000000-0000-4000-8000-000000000005"; +const userId = "00000000-0000-4000-8000-000000000006"; +const path = "/api/control-plane/runs/run-1/stream"; + +async function request(signal?: AbortSignal, requestPath = path) { + const body = JSON.stringify({ + run: { + agentServiceId: "service-1", + agentId: "builder", + conversationId: "00000000-0000-4000-8000-000000000001", + runId: "run-1", + messageId: "00000000-0000-4000-8000-000000000002", + inputAnchorMessageId: "00000000-0000-4000-8000-000000000003", + requestedByUserId: userId, + project: { projectId, projectSlug: "demo-project", runtimeTargetKind: "main_branch" }, + }, + messages: [], + tools: [], + context: [], + agentSource: { type: "release", releaseId: "release-1" }, + credentials: { authToken: "api-token", inferenceAuthToken: "inference-token" }, + }); + const signed = await createControlPlaneSignature(body, { + audience: "demo-project", + projectId, + requestId: "run-1", + requestPath, + }); + return { + publicKeyPem: signed.publicKeyPem, + request: new Request(`https://broker.test${requestPath}`, { + method: "POST", + headers: { + authorization: "Bearer broker-token", + "x-veryfront-control-plane-jws": signed.jws, + "x-veryfront-run-event-token": "event-token", + }, + body, + signal, + }), + }; +} + +function runtimeFixture( + streamFailure = false, + terminalChunk?: "error" | "coded-error" | "finish-error", + finishWithUsage?: true, +) { + const release = Promise.withResolvers(); + const settled = Promise.withResolvers(); + const acceptKinds: string[] = []; + const closeReasons: string[] = []; + let streamCalls = 0; + const runtime: ManagedExecutorRuntime = { + definition: { id: "builder", name: "Builder", description: "Builds", instructions: "Work" }, + modelId: "veryfront-cloud/openai/synthetic", + runtimeKind: "framework", + accepted: false, + settled: settled.promise, + runOwned: (operation) => operation(), + accept(ownership) { + acceptKinds.push(ownership.kind); + Object.defineProperty(runtime, "accepted", { value: true }); + }, + close(reason = "canceled") { + closeReasons.push(reason); + settled.resolve(); + return Promise.resolve({ reason, release: "released" }); + }, + agent: { + async stream() { + streamCalls++; + return { + steps: Promise.resolve([]), + toUIMessageStream(streamOptions = {}) { + return (async function* () { + await release.promise; + if (streamFailure) throw new Error("synthetic stream failure"); + if (terminalChunk === "error") { + yield { type: "error", errorText: "ordinary stream error" } as const; + return; + } + if (terminalChunk === "coded-error") { + yield { + type: "error", + errorText: "INSUFFICIENT_CREDITS", + code: "INSUFFICIENT_CREDITS", + } as const; + return; + } + if (terminalChunk === "finish-error") { + yield { type: "finish", finishReason: "error" } as const; + return; + } + if (finishWithUsage) { + const part = { + type: "finish" as const, + finishReason: "stop" as const, + totalUsage: { + inputTokens: 12, + outputTokens: 7, + usageCaptureStatus: "complete" as const, + }, + }; + yield { type: "text-delta", id: "assistant-message", delta: "done" } as const; + yield { + type: "finish", + finishReason: "stop", + messageMetadata: streamOptions.messageMetadata?.({ part }), + } as const; + return; + } + yield { type: "start", messageId: "assistant-message" } as const; + })(); + }, + }; + }, + }, + }; + return { + runtime, + release: release.resolve, + acceptKinds, + closeReasons, + get streamCalls() { + return streamCalls; + }, + }; +} + +async function handler( + mode: "detached" | "sse", + options: { + signal?: AbortSignal; + serviceSignal?: AbortSignal; + requestPath?: string; + failStart?: boolean; + admitBeforeFailure?: boolean; + waitForPrepare?: boolean; + abortPrepare?: boolean; + waitForAuthorization?: boolean; + throwingObserver?: boolean; + streamFailure?: boolean; + terminalChunk?: "error" | "coded-error" | "finish-error"; + finishWithUsage?: true; + missingOutput?: boolean; + waitForOutput?: boolean; + startError?: Error; + } = {}, +) { + const first = await request(options.signal, options.requestPath); + const fixture = runtimeFixture( + options.streamFailure, + options.terminalChunk, + options.finishWithUsage, + ); + let prepareCalls = 0; + let brokerStarts = 0; + let cleanupCalls = 0; + const outputChunks: string[] = []; + const outputFinishes: boolean[] = []; + const outputFinishErrors: unknown[] = []; + const outputFinishMetadata: unknown[] = []; + const outputRelease = Promise.withResolvers(); + let prepareSignal: AbortSignal | undefined; + const prepareEntered = Promise.withResolvers(); + const prepareRelease = Promise.withResolvers(); + const authorizationEntered = Promise.withResolvers(); + const authorizationRelease = Promise.withResolvers(); + const admitted = Promise.withResolvers(); + const executionController = new AbortController(); + const managed = createManagedBrokerHandler({ + responseMode: mode, + signal: options.serviceSignal, + broker: { + start: (start, lifecycle) => { + brokerStarts++; + prepareSignal = start.session.preparationSignal; + if (options.admitBeforeFailure) lifecycle?.onAdmitted?.(admitted.promise); + return options.failStart + ? Promise.reject(options.startError ?? new Error("synthetic start failure")) + : Promise.resolve(fixture.runtime); + }, + }, + resolveIngressOptions: () => ({ + publicKeyPem: first.publicKeyPem, + audience: "demo-project", + projectId, + expectedSurface: "studio", + boundSource: { type: "release", releaseId: "release-1" }, + expectedOwner: { scopeKind: "project", projectId }, + authorizeScope: async () => { + authorizationEntered.resolve(); + if (options.waitForAuthorization) await authorizationRelease.promise; + options.serviceSignal?.throwIfAborted(); + return { userId }; + }, + }), + prepare: async ({ signal }) => { + prepareCalls++; + prepareSignal = signal; + prepareEntered.resolve(); + if (options.waitForPrepare) await prepareRelease.promise; + if (options.abortPrepare) { + await new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + } + return { + start: { session: {} } as ManagedExecutorStartInput, + messages: [], + executionSignal: executionController.signal, + output: options.missingOutput ? undefined : { + async write(chunk: { type: string }) { + outputChunks.push(chunk.type); + }, + async finish(outcome: { completed: boolean; error?: unknown; metadata?: unknown }) { + outputFinishes.push(outcome.completed); + outputFinishErrors.push(outcome.error); + outputFinishMetadata.push(outcome.metadata); + if (options.waitForOutput) await outputRelease.promise; + }, + }, + cleanup: async () => { + cleanupCalls++; + }, + }; + }, + onExecutionError: options.throwingObserver + ? () => { + throw new Error("observer"); + } + : undefined, + }); + return { + first, + fixture, + managed, + admitted: admitted.resolve, + prepareEntered: prepareEntered.promise, + releasePrepare: prepareRelease.resolve, + authorizationEntered: authorizationEntered.promise, + releaseAuthorization: authorizationRelease.resolve, + outputChunks, + outputFinishes, + outputFinishErrors, + outputFinishMetadata, + releaseOutput: outputRelease.resolve, + abortExecution: () => executionController.abort(), + get prepareCalls() { + return prepareCalls; + }, + get brokerStarts() { + return brokerStarts; + }, + get cleanupCalls() { + return cleanupCalls; + }, + get prepareSignal() { + return prepareSignal; + }, + }; +} + +describe("managed broker handler", () => { + it("accepts encoded run IDs while verifying the original signed path", async () => { + const encodedPath = "/api/control-plane/runs/%72un%2D1/stream"; + const f = await handler("detached", { requestPath: encodedPath }); + try { + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 202); + assertEquals(f.brokerStarts, 1); + } finally { + f.fixture.release(); + await f.managed.close(); + } + const canonical = await handler("detached"); + try { + const original = canonical.first.request; + const changedPath = new Request(`https://broker.test${encodedPath}`, { + method: original.method, + headers: original.headers, + body: await original.text(), + }); + const response = await canonical.managed.handle(changedPath); + assertEquals(response.status, 401); + assertEquals(await response.json(), { errorCode: "BROKER_INGRESS_AUTH_INVALID" }); + assertEquals(canonical.brokerStarts, 0); + } finally { + canonical.fixture.release(); + await canonical.managed.close(); + } + }); + it("preserves typed executor failure statuses on both response modes", async () => { + for (const mode of ["detached", "sse"] as const) { + const f = await handler(mode, { + failStart: true, + startError: new ExecutorAgentError("INSUFFICIENT_CREDITS"), + }); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 402); + assertEquals(await response.json(), { errorCode: "INSUFFICIENT_CREDITS" }); + await f.managed.close(); + } + }); + it("requires a durable output writer before allocating a detached run", async () => { + const f = await handler("detached", { missingOutput: true }); + try { + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 500); + assertEquals(f.brokerStarts, 0); + } finally { + f.fixture.release(); + await f.managed.close(); + } + }); + + it("persists detached output and retains retirement until the final write settles", async () => { + const f = await handler("detached", { waitForOutput: true }); + try { + assertEquals((await f.managed.handle(f.first.request)).status, 202); + f.fixture.release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(f.outputChunks, ["start"]); + assertEquals(f.outputFinishes, [true]); + assertEquals(f.fixture.closeReasons, []); + assertEquals(f.managed.active, 1); + } finally { + f.releaseOutput(); + f.fixture.release(); + await f.managed.close(); + } + assertEquals(f.cleanupCalls, 1); + }); + it("preserves detached finish usage metadata for durable finalization", async () => { + const f = await handler("detached", { finishWithUsage: true }); + assertEquals((await f.managed.handle(f.first.request)).status, 202); + f.fixture.release(); + await f.managed.close(); + assertEquals(f.outputFinishMetadata, [{ + modelId: "veryfront-cloud/openai/synthetic", + usage: { inputTokens: 12, outputTokens: 7 }, + usageCaptureStatus: "complete", + }]); + }); + it("transfers detached ownership before 202 and prevents duplicate allocation", async () => { + const f = await handler("detached"); + const duplicateRequest = f.first.request.clone(); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 202); + assertEquals(await response.json(), { accepted: true, duplicate: false }); + assertEquals(f.fixture.acceptKinds, ["execution"]); + const duplicate = await f.managed.handle(duplicateRequest); + assertEquals(await duplicate.json(), { accepted: true, duplicate: true }); + assertEquals(f.prepareCalls, 1); + f.fixture.release(); + await f.managed.close(); + assertEquals(f.fixture.closeReasons, ["completed"]); + }); + + it("does not acknowledge a duplicate while the original run is still pending admission", async () => { + const f = await handler("detached", { waitForPrepare: true, failStart: true }); + const duplicateRequest = f.first.request.clone(); + const original = f.managed.handle(f.first.request); + await f.prepareEntered; + + const duplicate = await f.managed.handle(duplicateRequest); + assertEquals(duplicate.status, 409); + assertEquals(await duplicate.json(), { errorCode: "BROKER_RUN_PENDING" }); + assertEquals(f.prepareCalls, 1); + + f.releasePrepare(); + assertEquals((await original).status, 500); + assertEquals(f.managed.active, 0); + await f.managed.close(); + }); + + it("marks ordinary error chunks and error finish reasons as failed durable output", async () => { + for (const terminalChunk of ["error", "finish-error"] as const) { + const f = await handler("detached", { terminalChunk }); + assertEquals((await f.managed.handle(f.first.request)).status, 202); + f.fixture.release(); + await f.managed.close(); + + assertEquals(f.outputFinishes, [false], terminalChunk); + assertEquals(f.outputFinishErrors[0] instanceof Error, true, terminalChunk); + assertEquals(f.outputChunks, [terminalChunk === "error" ? "error" : "finish"]); + assertEquals(f.fixture.closeReasons, ["canceled"], terminalChunk); + } + }); + + it("preserves a validated executor error chunk through durable terminal classification", async () => { + const f = await handler("detached", { terminalChunk: "coded-error" }); + assertEquals((await f.managed.handle(f.first.request)).status, 202); + f.fixture.release(); + await f.managed.close(); + + assertEquals(resolveConversationHostedStreamErrorState(f.outputFinishErrors[0]), { + status: "failed", + terminalErrorCode: "INSUFFICIENT_CREDITS", + terminalErrorMessage: "Insufficient AI credits", + }); + }); + + it("finalizes an aborted detached execution as cancelled instead of failed", async () => { + const f = await handler("detached"); + assertEquals((await f.managed.handle(f.first.request)).status, 202); + f.abortExecution(); + f.fixture.release(); + await f.managed.close(); + + assertEquals(f.outputFinishes, [false]); + assertEquals(f.outputFinishErrors, [undefined]); + assertEquals(f.fixture.closeReasons, ["canceled"]); + }); + + it("preserves request-owned SSE and releases only after response completion", async () => { + const f = await handler("sse"); + const duplicateRequest = f.first.request.clone(); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 200); + assertEquals(response.headers.get("content-type"), "text/event-stream; charset=utf-8"); + assertEquals(f.fixture.acceptKinds, ["request"]); + assertEquals(f.managed.active, 1); + const duplicate = await f.managed.handle(duplicateRequest); + assertEquals(duplicate.status, 409); + assertEquals(f.prepareCalls, 1); + f.fixture.release(); + await response.text(); + await f.managed.close(); + assertEquals(f.fixture.closeReasons, ["completed"]); + assertEquals(f.managed.active, 0); + }); + + it("preserves finish usage metadata in the SSE RunFinished event", async () => { + const f = await handler("sse", { finishWithUsage: true }); + const response = await f.managed.handle(f.first.request); + f.fixture.release(); + const parsed = await parseAgUiSseResponse(response); + await f.managed.close(); + const finished = parsed.events.find((event) => event.type === agUiSseEventTypes.runFinished); + const metadata = finished?.metadata as Record | undefined; + + assertEquals({ + inputTokens: metadata?.inputTokens, + outputTokens: metadata?.outputTokens, + totalTokens: metadata?.totalTokens, + usageCaptureStatus: metadata?.usageCaptureStatus, + finishReason: metadata?.finishReason, + }, { + inputTokens: 12, + outputTokens: 7, + totalTokens: 19, + usageCaptureStatus: "complete", + finishReason: "stop", + }); + }); + + it("releases failed setup reservations and maps the error without diagnostics", async () => { + const f = await handler("detached", { failStart: true }); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 500); + assertEquals(await response.json(), { errorCode: "BROKER_EXECUTION_SETUP_FAILED" }); + assertEquals(f.managed.active, 0); + assertEquals(f.cleanupCalls, 1); + }); + + it("retains admitted setup failure and cleanup until actual session settlement", async () => { + const f = await handler("detached", { failStart: true, admitBeforeFailure: true }); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 500); + assertEquals(f.managed.active, 1); + assertEquals(f.cleanupCalls, 0); + f.admitted(); + await f.managed.close(); + assertEquals(f.cleanupCalls, 1); + assertEquals(f.managed.active, 0); + }); + + it("does not admit a prepared run after handler closure during preparation", async () => { + const f = await handler("detached", { waitForPrepare: true }); + const response = f.managed.handle(f.first.request); + await f.prepareEntered; + const closing = f.managed.close(); + assertEquals(f.prepareSignal?.aborted, true); + f.releasePrepare(); + assertEquals((await response).status, 503); + await closing; + assertEquals(f.brokerStarts, 0); + assertEquals(f.cleanupCalls, 1); + }); + + for (const mode of ["detached", "sse"] as const) { + for (const source of ["shutdown", "service", "request"] as const) { + it(`maps ${source} during abortable ${mode} preparation without a setup failure`, async () => { + const controller = new AbortController(); + const f = await handler(mode, { + signal: source === "request" ? controller.signal : undefined, + serviceSignal: source === "service" ? controller.signal : undefined, + abortPrepare: true, + }); + const response = f.managed.handle(f.first.request); + await f.prepareEntered; + if (source === "shutdown") void f.managed.close(); + else controller.abort(); + try { + const result = await response; + assertEquals(result.status, source === "request" ? 499 : 503); + assertEquals(await result.json(), { + errorCode: source === "request" ? "BROKER_INGRESS_ABORTED" : "BROKER_UNAVAILABLE", + }); + assertEquals(f.brokerStarts, 0); + assertEquals(f.managed.active, 0); + } finally { + await f.managed.close(); + } + }); + } + } + + for (const mode of ["detached", "sse"] as const) { + it(`maps service shutdown during ${mode} authorization to unavailable`, async () => { + const controller = new AbortController(); + const f = await handler(mode, { + serviceSignal: controller.signal, + waitForAuthorization: true, + }); + const response = f.managed.handle(f.first.request); + await f.authorizationEntered; + controller.abort(); + f.releaseAuthorization(); + try { + const result = await response; + assertEquals(result.status, 503); + assertEquals(await result.json(), { errorCode: "BROKER_UNAVAILABLE" }); + assertEquals(f.prepareCalls, 0); + } finally { + await f.managed.close(); + } + }); + + it(`maps service shutdown during ${mode} body reading to unavailable`, async () => { + const controller = new AbortController(); + const f = await handler(mode, { serviceSignal: controller.signal }); + const blockedBody = new ReadableStream({ + start(stream) { + stream.enqueue(new TextEncoder().encode("{")); + }, + }); + const pending = f.managed.handle( + new Request(f.first.request.url, { + method: "POST", + headers: f.first.request.headers, + body: blockedBody, + duplex: "half", + } as RequestInit), + ); + controller.abort(); + try { + const result = await pending; + assertEquals(result.status, 503); + assertEquals(await result.json(), { errorCode: "BROKER_UNAVAILABLE" }); + assertEquals(f.prepareCalls, 0); + } finally { + await f.managed.close(); + } + }); + } + + it("does not admit a late authorization result after handler closure", async () => { + const f = await handler("detached", { waitForAuthorization: true }); + const response = f.managed.handle(f.first.request); + await f.authorizationEntered; + await f.managed.close(); + f.releaseAuthorization(); + assertEquals((await response).status, 503); + assertEquals(f.prepareCalls, 0); + assertEquals(f.brokerStarts, 0); + }); + + it("closes request-owned SSE as cancelled after request abort", async () => { + const controller = new AbortController(); + const f = await handler("sse", { signal: controller.signal }); + const response = await f.managed.handle(f.first.request); + controller.abort(); + f.fixture.release(); + await response.text(); + await f.managed.close(); + assertEquals(f.fixture.closeReasons, ["canceled"]); + }); + + it("cancels and retires request-owned SSE when the response body is canceled", async () => { + const f = await handler("sse"); + const response = await f.managed.handle(f.first.request); + let closing: Promise | undefined; + try { + await response.body?.cancel("client stopped reading"); + closing = f.managed.close(); + const timeout = Promise.withResolvers(); + const timeoutId = setTimeout(() => timeout.resolve(false), 25); + const retired = await Promise.race([closing.then(() => true), timeout.promise]); + clearTimeout(timeoutId); + assertEquals(retired, true); + assertEquals(f.fixture.closeReasons, ["canceled"]); + assertEquals(f.managed.active, 0); + assertEquals(f.cleanupCalls, 1); + } finally { + f.fixture.release(); + await closing; + } + }); + + it("shields throwing detached execution observers", async () => { + const f = await handler("detached", { streamFailure: true, throwingObserver: true }); + const response = await f.managed.handle(f.first.request); + assertEquals(response.status, 202); + f.fixture.release(); + await f.managed.close(); + assertEquals(f.fixture.closeReasons, ["canceled"]); + }); +}); diff --git a/src/agent/service/managed-broker-handler.ts b/src/agent/service/managed-broker-handler.ts new file mode 100644 index 0000000000..cfa9f8573d --- /dev/null +++ b/src/agent/service/managed-broker-handler.ts @@ -0,0 +1,405 @@ +import type { + HostedChatRuntimeFinishPart, + HostedChatRuntimeStreamInput, +} from "../hosted/chat-runtime-contract.ts"; +import { + ExecutorAgentError, + getExecutorAgentFailureCodeSchema, +} from "../hosted/executor-agent-schema.ts"; +import { ExecutorRuntimePreparationError } from "../hosted/executor-runtime-prepare-schema.ts"; +import { ExecutorDiscoveryError } from "../hosted/executor-discovery-schema.ts"; +import { HostedServiceAuthError } from "./auth.ts"; +import { createAgUiChatUiTrackedResponse } from "../ag-ui/chat-ui-chunk-encoder.ts"; +import type { AgUiRuntimeRequest } from "../runtime/ag-ui-contract.ts"; +import { buildChatStreamChunkMessageMetadata } from "../../chat/chat-ui-message-helpers.ts"; +import type { HostedLifecycleTerminalState } from "../hosted/lifecycle.ts"; +import type { + ManagedExecutorRuntime, + ManagedExecutorStartInput, +} from "../hosted/managed-executor-broker.ts"; +import type { ManagedBrokerOutput } from "../hosted/managed-broker-persistence.ts"; +import { + BrokerIngressError, + type BrokerRuntimeAgentIngress, + type BrokerRuntimeAgentIngressOptions, + parseBrokerRuntimeAgentIngress, +} from "./broker-ingress.ts"; + +import { parseBrokerSignedRunPath } from "./broker-run-route.ts"; + +/** Trusted executor admission boundary with actual settlement notification. */ +export interface ManagedExecutorStarter { + start( + input: ManagedExecutorStartInput, + lifecycle?: { onAdmitted?(settled: Promise): void }, + ): Promise; +} + +/** Handle signed run invocations with configured detached or request-owned SSE responses. */ +export function createManagedBrokerHandler(options: { + broker: ManagedExecutorStarter; + /** Trusted route configuration; never read from request data. */ + responseMode: "detached" | "sse"; + signal?: AbortSignal; + resolveIngressOptions(input: { + request: Request; + runId: string; + }): Omit, "expectedRunId">; + prepare(input: { + ingress: BrokerRuntimeAgentIngress; + signal: AbortSignal; + }): Promise<{ + start: ManagedExecutorStartInput; + messages: HostedChatRuntimeStreamInput["messages"]; + executionSignal: AbortSignal; + output?: ManagedBrokerOutput; + cleanup?: () => Promise; + }>; + onExecutionError?: (error: unknown, runId: string) => void; +}) { + const active = new Map }>(); + const lifetime = new AbortController(); + let closed = false; + + async function handle(request: Request): Promise { + const runId = parseBrokerSignedRunPath(new URL(request.url).pathname); + if (request.method !== "POST" || runId === null) { + return Response.json({ errorCode: "BROKER_INGRESS_TARGET_MISMATCH" }, { status: 400 }); + } + if (closed || options.signal?.aborted) { + return Response.json({ errorCode: "BROKER_UNAVAILABLE" }, { status: 503 }); + } + try { + const signal = AbortSignal.any([ + request.signal, + lifetime.signal, + ...(options.signal ? [options.signal] : []), + ]); + const ingress = await parseBrokerRuntimeAgentIngress(request, { + ...options.resolveIngressOptions({ request, runId }), + expectedRunId: runId, + signal, + }); + assertAvailable(closed, signal); + const runKey = managedRunKey(ingress); + const existing = active.get(runKey); + if (existing) { + if (!existing.accepted) { + return Response.json({ errorCode: "BROKER_RUN_PENDING" }, { status: 409 }); + } + return options.responseMode === "detached" + ? Response.json({ accepted: true, duplicate: true }, { status: 202 }) + : Response.json({ errorCode: "BROKER_RUN_ALREADY_ACTIVE" }, { status: 409 }); + } + const reservation = Promise.withResolvers(); + const activeRun = { accepted: false, settled: reservation.promise }; + active.set(runKey, activeRun); + let preparedCleanup: (() => Promise) | undefined; + let admissionSettled: Promise | undefined; + let retirement: Promise | undefined; + const release = () => { + if (active.get(runKey) === activeRun) active.delete(runKey); + reservation.resolve(); + }; + const retire = (settled: Promise = Promise.resolve()) => { + retirement ??= Promise.resolve().then(async () => { + await settled.catch(() => {}); + await preparedCleanup?.(); + }).finally(release); + return retirement; + }; + try { + const prepared = await options.prepare({ ingress, signal }); + preparedCleanup = prepared.cleanup; + assertAvailable(closed, signal); + if ( + options.responseMode === "detached" && + (typeof prepared.output?.write !== "function" || + typeof prepared.output?.finish !== "function") + ) { + throw new Error("Detached broker output persistence is required"); + } + const runtime = await options.broker.start({ + ...prepared.start, + session: { ...prepared.start.session, preparationSignal: signal }, + }, { + onAdmitted(settled) { + admissionSettled = settled; + }, + }); + admissionSettled = runtime.settled; + try { + runtime.accept( + options.responseMode === "detached" + ? { kind: "execution", signal: prepared.executionSignal } + : { kind: "request" }, + ); + activeRun.accepted = true; + } catch (error) { + await runtime.close("canceled").catch(() => {}); + void retire(runtime.settled).catch(() => {}); + throw error; + } + if (options.responseMode === "sse") { + try { + return await createSseResponse({ + runtime, + messages: prepared.messages, + requestSignal: request.signal, + runId, + threadId: ingress.executor.run.conversationId, + agentId: ingress.executor.run.agentId, + agUiInput: ingress.executor.input, + onSettled: () => retire(runtime.settled), + }); + } catch (error) { + await runtime.close("canceled").catch(() => {}); + void retire(runtime.settled).catch(() => {}); + throw error; + } + } + const execution = runDetached( + runtime, + prepared.messages, + prepared.executionSignal, + prepared.output!, + ) + .catch((error) => { + try { + options.onExecutionError?.(error, runId); + } catch { /* Observability cannot own execution settlement. */ } + }).finally(() => { + void retire(runtime.settled).catch(() => {}); + }); + void execution; + return Response.json({ accepted: true, duplicate: false }, { status: 202 }); + } catch (error) { + const retiring = retire(admissionSettled); + if (!admissionSettled) await retiring.catch(() => {}); + else void retiring.catch(() => {}); + throw error; + } + } catch (error) { + if ( + error instanceof BrokerHandlerUnavailableError || lifetime.signal.aborted || + options.signal?.aborted + ) { + return Response.json({ errorCode: "BROKER_UNAVAILABLE" }, { status: 503 }); + } + if (error instanceof BrokerIngressError) { + return Response.json({ errorCode: error.errorCode }, { status: error.status }); + } + const aborted = request.signal.aborted || options.signal?.aborted; + if (!aborted) { + if ( + error instanceof ExecutorAgentError || error instanceof ExecutorRuntimePreparationError || + error instanceof ExecutorDiscoveryError + ) { + return Response.json({ errorCode: error.code }, { status: error.status }); + } + if (error instanceof HostedServiceAuthError) { + return Response.json({ errorCode: error.errorCode }, { status: error.statusCode }); + } + } + return Response.json( + { errorCode: aborted ? "BROKER_INGRESS_ABORTED" : "BROKER_EXECUTION_SETUP_FAILED" }, + { status: aborted ? 499 : 500 }, + ); + } + } + + async function close(): Promise { + closed = true; + lifetime.abort(); + await Promise.allSettled([...active.values()].map((run) => run.settled)); + } + + return { + handle, + close, + get active() { + return active.size; + }, + }; +} + +/** Private lifecycle sentinel mapped locally to a fixed HTTP error without serializing diagnostics. */ +class BrokerHandlerUnavailableError extends Error {} + +function assertAvailable(closed: boolean, signal: AbortSignal): void { + if (closed) throw new BrokerHandlerUnavailableError(); + signal.throwIfAborted(); +} + +function managedRunKey(ingress: BrokerRuntimeAgentIngress): string { + const owner = ingress.executor.owner; + const ownerKey = owner.scopeKind === "project" + ? `project:${owner.projectId}` + : `global:${owner.serviceName}`; + return `${ownerKey}:${ingress.executor.run.project.projectId}:${ingress.executor.run.runId}`; +} + +function buildManagedBrokerMessageMetadata( + runtime: ManagedExecutorRuntime, + part: HostedChatRuntimeFinishPart, +) { + return buildChatStreamChunkMessageMetadata({ + agentId: runtime.definition.id, + agentName: runtime.definition.name, + agentAvatarUrl: runtime.definition.avatarUrl, + modelId: runtime.modelId, + part: { type: part.type, totalUsage: part.totalUsage }, + }); +} + +async function createSseResponse(input: { + runtime: ManagedExecutorRuntime; + messages: HostedChatRuntimeStreamInput["messages"]; + requestSignal: AbortSignal; + runId: string; + threadId: string; + agentId: string; + agUiInput: AgUiRuntimeRequest; + onSettled(): Promise; +}): Promise { + const result = await input.runtime.agent.stream({ + messages: input.messages, + abortSignal: input.requestSignal, + }); + const source = result.toUIMessageStream({ + messageMetadata: ({ part }) => buildManagedBrokerMessageMetadata(input.runtime, part), + }); + const completion = Promise.withResolvers(); + let natural = false; + let cleanup: Promise | undefined; + const finish = (reason: "completed" | "canceled") => { + cleanup ??= Promise.resolve().then(async () => { + await input.runtime.close(reason).catch(() => {}); + await input.runtime.settled; + await input.onSettled(); + }); + return cleanup; + }; + const agentUIStream = (async function* () { + try { + for await (const chunk of source) yield chunk; + natural = true; + } finally { + completion.resolve(); + } + })(); + const response = createAgUiChatUiTrackedResponse({ + agUiInput: input.agUiInput, + defaults: { runId: input.runId, threadId: input.threadId }, + agentId: input.agentId, + modelId: input.runtime.modelId, + execution: { + agentUIStream, + async fail() { + await finish("canceled"); + }, + async waitForFinish() { + await completion.promise; + await finish(natural && !input.requestSignal.aborted ? "completed" : "canceled"); + }, + }, + }); + return withResponseBodyCancellation(response, () => finish("canceled")); +} + +function withResponseBodyCancellation( + response: Response, + cancel: () => Promise, +): Response { + if (!response.body) return response; + const reader = response.body.getReader(); + const body = new ReadableStream({ + async pull(controller) { + try { + const result = await reader.read(); + if (result.done) { + controller.close(); + reader.releaseLock(); + return; + } + controller.enqueue(result.value); + } catch (error) { + await cancel().catch(() => {}); + controller.error(error); + } + }, + async cancel(reason) { + await Promise.allSettled([ + reader.cancel(reason), + cancel(), + ]); + }, + }); + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function runDetached( + runtime: ManagedExecutorRuntime, + messages: HostedChatRuntimeStreamInput["messages"], + signal: AbortSignal, + output: ManagedBrokerOutput, +): Promise { + let completed = false; + try { + await runtime.runOwned(async () => { + let streamCompleted = false; + let failure: unknown; + let terminalMetadata: HostedLifecycleTerminalState["metadata"]; + try { + const result = await runtime.agent.stream({ messages, abortSignal: signal }); + for await ( + const chunk of result.toUIMessageStream({ + messageMetadata({ part }) { + const metadata = buildManagedBrokerMessageMetadata(runtime, part); + terminalMetadata = { + modelId: metadata.modelId, + ...(metadata.usage ? { usage: metadata.usage } : {}), + ...(metadata.usageCaptureStatus + ? { usageCaptureStatus: metadata.usageCaptureStatus } + : {}), + }; + return metadata; + }, + }) + ) { + await output.write(chunk); + if (chunk.type === "error" && failure === undefined) { + const code = getExecutorAgentFailureCodeSchema().safeParse(chunk.code); + failure = code.success + ? new ExecutorAgentError(code.data) + : new Error(chunk.errorText || "Agent stream failed"); + } else if ( + chunk.type === "finish" && chunk.finishReason === "error" && failure === undefined + ) { + failure = new Error("Agent stream finished with an error"); + } + } + signal.throwIfAborted(); + if (failure !== undefined) throw failure; + streamCompleted = true; + } catch (error) { + failure ??= error; + throw error; + } finally { + await output.finish({ + completed: streamCompleted, + ...(failure === undefined || signal.aborted ? {} : { error: failure }), + ...(terminalMetadata ? { metadata: terminalMetadata } : {}), + }); + } + }); + completed = true; + } finally { + await runtime.close(completed ? "completed" : "canceled").catch(() => {}); + await runtime.settled; + } +} diff --git a/src/agent/service/managed-broker.ts b/src/agent/service/managed-broker.ts new file mode 100644 index 0000000000..3cabce3765 --- /dev/null +++ b/src/agent/service/managed-broker.ts @@ -0,0 +1,43 @@ +/** Managed broker composition without project runtime or application imports. */ +export { + createManagedExecutorBroker, + type ManagedExecutorBrokerOptions, + type ManagedExecutorRuntime, + type ManagedExecutorStartInput, +} from "../hosted/managed-executor-broker.ts"; +export { + createManagedBrokerPersistence, + type ManagedBrokerOutput, +} from "../hosted/managed-broker-persistence.ts"; +export { createManagedBrokerProjectState } from "../hosted/managed-broker-project-state.ts"; +export { createHostedExecutorAllocatorClient } from "../hosted/executor-allocator-client.ts"; +export { + connectExecutorTransport, + type ConnectExecutorTransportOptions, +} from "../hosted/executor-node-transport.ts"; +export { + createManagedBrokerHandler, + type ManagedExecutorStarter, +} from "./managed-broker-handler.ts"; +export { + BrokerIngressError, + type BrokerIngressErrorCode, + type BrokerIngressScopeInput, + type BrokerRuntimeAgentExecutorInput, + type BrokerRuntimeAgentIngress, + type BrokerRuntimeAgentIngressOptions, + type BrokerRuntimeAgentPrivateAuthority, + parseBrokerRuntimeAgentIngress, +} from "./broker-ingress.ts"; +export { + type ManagedAgentBrokerIngressAuthority, + type ManagedAgentExecutorRequest, + type ManagedAgentIngressResult, + parseManagedAgUiAgentIngress, + parseManagedDurableAgentIngress, +} from "./managed-hosted-ingress.ts"; +export { + type ManagedNodeBrokerHandler, + type ManagedNodeBrokerPool, + startNodeManagedAgentBroker, +} from "./managed-node-broker.ts"; diff --git a/src/agent/service/managed-hosted-ingress.test.ts b/src/agent/service/managed-hosted-ingress.test.ts new file mode 100644 index 0000000000..2b60fb0da2 --- /dev/null +++ b/src/agent/service/managed-hosted-ingress.test.ts @@ -0,0 +1,265 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { + parseManagedAgUiAgentIngress, + parseManagedDurableAgentIngress, +} from "./managed-hosted-ingress.ts"; + +const projectId = "10000000-1000-4000-8000-100000000005"; +const conversationId = "10000000-1000-4000-8000-100000000001"; +const messageId = "10000000-1000-4000-8000-100000000002"; + +function authenticate() { + return Promise.resolve({ userId: "user-1", authToken: "broker-auth-secret" }); +} + +function verifyProjectAccess() { + return Promise.resolve({ success: true as const, projectSlug: "demo" }); +} + +describe("managed agent ingress", () => { + for (const kind of ["durable", "ag-ui"] as const) { + for ( + const credential of [ + "broker-auth-secret", + "request-auth-secret", + "run-event-secret", + "inference-secret", + ] + ) { + for ( + const placement of [ + "message", + "url", + "property name", + "encoded url", + "encoded property name", + ] as const + ) { + it(`rejects ${credential} embedded in ${kind} ${placement}`, async () => { + const text = placement === "message" ? `Use ${credential} for this request` : "Hello"; + const visibleCredential = placement.startsWith("encoded") + ? "%ZZ" + credential.replaceAll("-", "%252D") + : credential; + const extra = placement === "url" || placement === "encoded url" + ? { + attachments: [{ + url: `https://files.test/document?token=${visibleCredential}&download=1`, + }], + } + : placement === "property name" || placement === "encoded property name" + ? { [`result-${visibleCredential}-metadata`]: "value" } + : {}; + const payload = kind === "durable" + ? { + messages: [{ id: "m1", role: "user", parts: [{ type: "text", text }] }], + context: { conversationId, projectId, branchId: "branch-1" }, + durableRootRun: { runId: "run_root_1", messageId }, + forwardedProps: extra, + } + : { + threadId: conversationId, + runId: "run-1", + messages: [{ id: "m1", role: "user", content: text }], + tools: [], + context: [{ description: "veryfront.projectId", value: JSON.stringify(projectId) }], + state: extra, + }; + const request = new Request(`https://agent.example.test/api/${kind}`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer request-auth-secret", + "X-Veryfront-Run-Event-Token": "run-event-secret", + "X-Veryfront-Inference-Token": "inference-secret", + }, + body: JSON.stringify(payload), + }); + const options = { + authenticate, + verifyProjectAccess, + verifyRunEventAppendToken: () => Promise.resolve(true), + }; + const result = kind === "durable" + ? await parseManagedDurableAgentIngress(request, options) + : await parseManagedAgUiAgentIngress(request, options); + assert(result instanceof Response); + assertEquals(result.status, 403); + assertEquals(await result.json(), { errorCode: "BROKER_INGRESS_SCOPE_DENIED" }); + }); + } + } + } + + for (const verify of [false, true]) { + it(`admits durable ingress without a resolved project slug (verify=${verify})`, async () => { + const request = new Request("https://agent.example.test/api/runs", { + method: "POST", + headers: { + "content-type": "application/json", + "X-Veryfront-Run-Event-Token": "run-event-secret", + }, + body: JSON.stringify({ + messages: [{ id: "m1", role: "user", parts: [{ type: "text", text: "Hello" }] }], + context: { conversationId, branchId: "branch-1", projectId }, + durableRootRun: { runId: "run_root_1", messageId }, + }), + }); + const result = await parseManagedDurableAgentIngress(request, { + authenticate, + ...(verify + ? { verifyProjectAccess: () => Promise.resolve({ success: true as const }) } + : {}), + verifyRunEventAppendToken: () => Promise.resolve(true), + }); + if (result instanceof Response) throw new Error("Expected managed durable ingress"); + assertEquals(result.executor.projectSlug, null); + assert( + result.executor.context !== null && typeof result.executor.context === "object" && + !Array.isArray(result.executor.context), + ); + assertEquals(Object.hasOwn(result.executor.context, "projectSlug"), false); + assertEquals(result.executor.context.conversationId, conversationId); + assertEquals(result.executor.projectId, projectId); + }); + } + + it("separates durable broker authority from a detached executor request", async () => { + const request = new Request("https://agent.example.test/api/runs", { + method: "POST", + headers: { + "content-type": "application/json", + "X-Veryfront-Run-Event-Token": "run-event-secret", + "X-Veryfront-Inference-Token": "inference-secret", + }, + body: JSON.stringify({ + messages: [{ id: "m1", role: "user", parts: [{ type: "text", text: "Hello" }] }], + context: { conversationId, projectId, branchId: "branch-1" }, + durableRootRun: { runId: "run_root_1", messageId }, + }), + }); + + const result = await parseManagedDurableAgentIngress(request, { + authenticate, + verifyProjectAccess, + verifyRunEventAppendToken: () => Promise.resolve(true), + }); + if (result instanceof Response) throw new Error("Expected managed durable ingress"); + + assertEquals(result.kind, "durable"); + assertEquals(result.executor.kind, "durable"); + assertEquals(result.executor.projectId, projectId); + assertEquals(result.executor.projectSlug, "demo"); + assertEquals(result.executor.durableRootRun, { runId: "run_root_1", messageId }); + assertEquals(result.executor.serverEnvelopeVerified, false); + assertEquals( + JSON.stringify(result), + JSON.stringify({ + kind: "durable", + broker: {}, + executor: result.executor, + }), + ); + + const serialized = JSON.stringify(result.executor); + for (const secret of ["broker-auth-secret", "run-event-secret", "inference-secret"]) { + assertEquals(serialized.includes(secret), false); + } + for (const forbiddenKey of ["authToken", "authorization", "rawRequest", "headers"]) { + assertEquals(forbiddenKey in result.executor, false); + } + + assertEquals(typeof result.broker.createInferenceModelResolver(), "function"); + assertEquals( + typeof result.broker.createRunEventWriterCapability({ + apiUrl: "https://api.example.test", + })?.mintChildRunEventWriterCapability, + "function", + ); + assertEquals(result.broker.getParsedRequest().authToken, "broker-auth-secret"); + + const parsed = result.broker.getParsedRequest(); + const originalText = parsed.messages[0]?.parts[0]; + if (originalText && typeof originalText === "object" && "text" in originalText) { + originalText.text = "mutated after projection"; + } + assertEquals(JSON.stringify(result.executor).includes("mutated after projection"), false); + }); + + it("preserves AG-UI validation shape and keeps unverified replay state out of the executor", async () => { + const invalid = await parseManagedAgUiAgentIngress( + new Request("https://agent.example.test/api/ag-ui", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ runId: "missing-thread-and-messages" }), + }), + { authenticate, verifyProjectAccess }, + ); + if (!(invalid instanceof Response)) throw new Error("Expected validation response"); + assertEquals(invalid.status, 400); + const invalidBody = await invalid.json(); + assertEquals((invalidBody as { errorCode?: unknown }).errorCode, "VALIDATION_ERROR"); + assertEquals(typeof (invalidBody as { message?: unknown }).message, "string"); + + const result = await parseManagedAgUiAgentIngress( + new Request("https://agent.example.test/api/ag-ui", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + threadId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + messages: [{ id: "u1", role: "user", content: "Hello" }], + tools: [], + context: [{ description: "veryfront.projectId", value: JSON.stringify(projectId) }], + forwardedProps: { + serverResolvedProviderReplayCheckpoints: { forged: "replay-secret" }, + }, + serverResolvedProviderReplayCheckpoints: { forged: "top-level-replay-secret" }, + }), + }), + { authenticate, verifyProjectAccess }, + ); + if (result instanceof Response) throw new Error("Expected managed AG-UI ingress"); + + assertEquals(result.kind, "ag-ui"); + assertEquals(result.executor.kind, "ag-ui"); + assertEquals(result.executor.agUi, { + threadId: "11111111-1111-4111-8111-111111111111", + runId: "run-1", + parentRunId: null, + tools: [], + context: [{ description: "veryfront.projectId", value: JSON.stringify(projectId) }], + }); + const serialized = JSON.stringify(result.executor); + assertEquals(serialized.includes("broker-auth-secret"), false); + assertEquals(serialized.includes("replay-secret"), false); + assertEquals(result.broker.createInferenceModelResolver(), undefined); + assertEquals( + result.broker.createRunEventWriterCapability({ apiUrl: "https://api.example.test" }), + undefined, + ); + }); + + it("authenticates AG-UI before reading the request body", async () => { + const request = new Request("https://agent.example.test/api/ag-ui", { + method: "POST", + headers: { "X-Veryfront-Run-Event-Token": "must-stay-broker-private" }, + body: "not json", + }); + const response = new Response("unauthenticated", { status: 401 }); + const result = await parseManagedAgUiAgentIngress(request, { + authenticate: (applicationRequest) => { + assertEquals(applicationRequest.headers.get("X-Veryfront-Run-Event-Token"), null); + return Promise.resolve(response); + }, + }); + + assertEquals(result, response); + assertEquals(request.bodyUsed, false); + assertEquals( + request.headers.get("X-Veryfront-Run-Event-Token"), + "must-stay-broker-private", + ); + }); +}); diff --git a/src/agent/service/managed-hosted-ingress.ts b/src/agent/service/managed-hosted-ingress.ts new file mode 100644 index 0000000000..6cdf9d13e1 --- /dev/null +++ b/src/agent/service/managed-hosted-ingress.ts @@ -0,0 +1,307 @@ +import { containsBrokerCredential } from "#veryfront/agent/service/broker-credentials.ts"; +import type { JsonValue } from "#veryfront/schemas/index.ts"; +import { snapshotBoundedJsonValue } from "#veryfront/schemas/json-value.ts"; +import { + createApplicationRequest, + isInfrastructureOnlyRequestHeader, +} from "#veryfront/security/http/application-request.ts"; +import { + buildParsedHostedAgUiRequest, + createHostedAgUiValidationErrorResponse, + type ParsedHostedAgUiRequest, +} from "../hosted/ag-ui-chat-request.ts"; +import { + createHostedRunEventWriterCapabilityForRequest, + type HostedRunEventWriterCapability, +} from "../hosted/child-run-event-writer-token.ts"; +import { + INFERENCE_TOKEN_HEADER, + type ParsedHostedChatRequest, + parseHostedChatRequestFromRequest, + type ParseHostedChatRequestOptions, + RUN_EVENT_APPEND_TOKEN_HEADER, +} from "../hosted/chat-request-parser.ts"; +import { createHostedInferenceModelResolver } from "../hosted/inference-credential.ts"; +import type { AgentModelRuntimeResolver } from "../runtime/model-transport.ts"; +import { parseAgUiRuntimeRequestOrError } from "../runtime/ag-ui-contract.ts"; +import { isResponseLike } from "./response-like.ts"; + +export type ManagedAgentIngressKind = "durable" | "ag-ui"; + +export interface ManagedRunEventWriterCapabilityOptions { + apiUrl: string; + timeoutMs?: number; + fetch?: typeof globalThis.fetch; +} + +/** Private parsed request and credentials available only to trusted broker preparation. */ +export interface ManagedAgentBrokerIngressAuthority< + TRequest extends ParsedHostedChatRequest = ParsedHostedChatRequest, +> { + getParsedRequest(): TRequest; + createInferenceModelResolver(options?: { apiBaseUrl?: string }): + | AgentModelRuntimeResolver + | undefined; + createRunEventWriterCapability( + options: ManagedRunEventWriterCapabilityOptions, + ): HostedRunEventWriterCapability | undefined; +} + +export type ManagedAgentExecutorAgUiState = Readonly<{ + threadId: string; + runId: string; + parentRunId: string | null; + tools: JsonValue; + context: JsonValue; + state?: JsonValue; +}>; + +/** Detached bounded application data, without HTTP objects or broker credentials. */ +export type ManagedAgentExecutorRequest = Readonly<{ + protocolVersion: 1; + kind: ManagedAgentIngressKind; + agentId: string | null; + userId: string; + messages: JsonValue; + context: JsonValue; + projectId: string | null; + projectSlug: string | null; + conversationId: string | null; + parentRunId: string | null; + upstreamParentConversationId: string | null; + upstreamParentRunId: string | null; + spawnedFromToolCallId: string | null; + model: string | null; + allowDelegation: boolean | null; + forwardedProps: JsonValue; + runtimeOverrides: JsonValue; + durableRootRun: JsonValue; + persistLatestUserMessageBeforeDurableRun: boolean; + serverEnvelopeVerified: boolean; + serverResolvedIntegrationToolNames: JsonValue; + serverResolvedProviderReplayCheckpoints?: JsonValue; + agUi?: ManagedAgentExecutorAgUiState; +}>; + +export type ManagedDurableAgentIngressResult = Readonly<{ + kind: "durable"; + broker: ManagedAgentBrokerIngressAuthority; + executor: ManagedAgentExecutorRequest & { kind: "durable" }; +}>; + +export type ManagedAgUiAgentIngressResult = Readonly<{ + kind: "ag-ui"; + broker: ManagedAgentBrokerIngressAuthority; + executor: ManagedAgentExecutorRequest & { + kind: "ag-ui"; + agUi: ManagedAgentExecutorAgUiState; + }; +}>; + +/** Preserve the distinct durable and direct AG-UI ingress contracts. */ +export type ManagedAgentIngressResult = + | ManagedDurableAgentIngressResult + | ManagedAgUiAgentIngressResult; + +export type ParseManagedAgUiAgentIngressOptions = + & Pick< + ParseHostedChatRequestOptions, + "authenticate" | "verifyProjectAccess" + > + & { forwardedConfigNamespace?: string }; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function sanitizeForwardedProps(value: unknown): unknown { + if (!isRecord(value)) return null; + const sanitized: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!key.startsWith("serverResolved")) sanitized[key] = entry; + } + const runtimeOverrides = sanitized.runtimeOverrides; + if ( + isRecord(runtimeOverrides) && Object.hasOwn(runtimeOverrides, "serverResolvedIntegrationTools") + ) { + const sanitizedRuntimeOverrides = Object.fromEntries( + Object.entries(runtimeOverrides).filter(([key]) => key !== "serverResolvedIntegrationTools"), + ); + if (Object.keys(sanitizedRuntimeOverrides).length > 0) { + sanitized.runtimeOverrides = sanitizedRuntimeOverrides; + } else { + delete sanitized.runtimeOverrides; + } + } + return Object.keys(sanitized).length > 0 ? sanitized : null; +} + +/** Bun retains source headers when Request init supplies a replacement list. */ +function removeRetainedInfrastructureHeaders(request: Request): Request { + const headers = request.headers; + for (const name of [...headers.keys()]) { + if (isInfrastructureOnlyRequestHeader(name)) headers.delete(name); + } + return request; +} + +function boundedExecutorRequest(value: unknown): ManagedAgentExecutorRequest { + const snapshot = snapshotBoundedJsonValue(value); + if (!snapshot.success) { + throw new TypeError("Managed agent executor request must contain bounded JSON data"); + } + return snapshot.value as ManagedAgentExecutorRequest; +} + +function createBrokerAuthority( + parsedRequest: TRequest, +): ManagedAgentBrokerIngressAuthority { + const authority = Object.create(null) as ManagedAgentBrokerIngressAuthority; + Object.defineProperties(authority, { + getParsedRequest: { + enumerable: false, + value: () => parsedRequest, + }, + createInferenceModelResolver: { + enumerable: false, + value: (options?: { apiBaseUrl?: string }) => + createHostedInferenceModelResolver(parsedRequest, options), + }, + createRunEventWriterCapability: { + enumerable: false, + value: (options: ManagedRunEventWriterCapabilityOptions) => { + const runId = parsedRequest.durableRootRun?.runId; + return runId + ? createHostedRunEventWriterCapabilityForRequest(parsedRequest, { + ...options, + runId, + }) + : undefined; + }, + }, + }); + return Object.freeze(authority); +} + +function requestCredentials(request: Request): (string | null)[] { + return [ + request.headers.get("authorization"), + request.headers.get(RUN_EVENT_APPEND_TOKEN_HEADER), + request.headers.get(INFERENCE_TOKEN_HEADER), + ]; +} + +function createExecutorRequest( + kind: ManagedAgentIngressKind, + parsedRequest: ParsedHostedChatRequest, + credentials: readonly (string | null)[], + agUiInput?: ParsedHostedAgUiRequest["agUiInput"], +): ManagedAgentExecutorRequest | Response { + const { projectSlug, ...context } = parsedRequest.validatedContext; + const request = { + protocolVersion: 1, + kind, + agentId: parsedRequest.agentId ?? null, + userId: parsedRequest.userId, + messages: parsedRequest.messages, + context: projectSlug === undefined ? context : { ...context, projectSlug }, + projectId: parsedRequest.projectId, + projectSlug: parsedRequest.projectSlug ?? null, + conversationId: parsedRequest.conversationId ?? null, + parentRunId: parsedRequest.parentRunId ?? null, + upstreamParentConversationId: parsedRequest.upstreamParentConversationId ?? null, + upstreamParentRunId: parsedRequest.upstreamParentRunId ?? null, + spawnedFromToolCallId: parsedRequest.spawnedFromToolCallId ?? null, + model: parsedRequest.model ?? null, + allowDelegation: parsedRequest.allowDelegation ?? null, + forwardedProps: sanitizeForwardedProps(parsedRequest.forwardedProps), + runtimeOverrides: parsedRequest.runtimeOverrides ?? null, + durableRootRun: parsedRequest.durableRootRun ?? null, + persistLatestUserMessageBeforeDurableRun: + parsedRequest.persistLatestUserMessageBeforeDurableRun, + serverEnvelopeVerified: parsedRequest.serverEnvelopeVerified === true, + serverResolvedIntegrationToolNames: parsedRequest.serverResolvedIntegrationToolNames ?? [], + ...(parsedRequest.serverEnvelopeVerified === true && + Object.hasOwn(parsedRequest, "serverResolvedProviderReplayCheckpoints") + ? { + serverResolvedProviderReplayCheckpoints: + parsedRequest.serverResolvedProviderReplayCheckpoints, + } + : {}), + ...(agUiInput + ? { + agUi: { + threadId: agUiInput.threadId, + runId: agUiInput.runId, + parentRunId: agUiInput.parentRunId ?? null, + tools: agUiInput.tools, + context: agUiInput.context, + ...(Object.hasOwn(agUiInput, "state") && agUiInput.state !== undefined + ? { state: agUiInput.state } + : {}), + }, + } + : {}), + }; + const executor = boundedExecutorRequest(request); + if (containsBrokerCredential(executor, [parsedRequest.authToken, ...credentials])) { + return Response.json({ errorCode: "BROKER_INGRESS_SCOPE_DENIED" }, { status: 403 }); + } + return executor; +} + +/** Parse the trusted broker's direct canonical durable-run ingress. */ +export async function parseManagedDurableAgentIngress( + request: Request, + options: ParseHostedChatRequestOptions, +): Promise { + const credentials = requestCredentials(request); + const parsedRequest = await parseHostedChatRequestFromRequest(request, options); + if (isResponseLike(parsedRequest)) return parsedRequest; + const executor = createExecutorRequest("durable", parsedRequest, credentials); + if (isResponseLike(executor)) return executor; + return Object.freeze({ + kind: "durable" as const, + broker: createBrokerAuthority(parsedRequest), + executor: executor as + & ManagedAgentExecutorRequest + & { kind: "durable" }, + }); +} + +/** Parse the trusted broker's direct request-owned AG-UI ingress. */ +export async function parseManagedAgUiAgentIngress( + request: Request, + options: ParseManagedAgUiAgentIngressOptions, +): Promise { + const credentials = requestCredentials(request); + const applicationRequest = removeRetainedInfrastructureHeaders( + createApplicationRequest(request), + ); + const principal = await options.authenticate(applicationRequest); + if (isResponseLike(principal)) return principal; + + const agUiInput = await parseAgUiRuntimeRequestOrError(applicationRequest); + if (isResponseLike(agUiInput)) { + return await createHostedAgUiValidationErrorResponse(agUiInput); + } + + const parsedRequest = await buildParsedHostedAgUiRequest({ + agUiInput, + authToken: principal.authToken, + userId: principal.userId, + forwardedConfigNamespace: options.forwardedConfigNamespace, + verifyProjectAccess: options.verifyProjectAccess, + }); + if (isResponseLike(parsedRequest)) return parsedRequest; + const executor = createExecutorRequest("ag-ui", parsedRequest, credentials, agUiInput); + if (isResponseLike(executor)) return executor; + + return Object.freeze({ + kind: "ag-ui" as const, + broker: createBrokerAuthority(parsedRequest), + executor: executor as + & ManagedAgentExecutorRequest + & { kind: "ag-ui"; agUi: ManagedAgentExecutorAgUiState }, + }); +} diff --git a/src/agent/service/managed-node-broker.ts b/src/agent/service/managed-node-broker.ts new file mode 100644 index 0000000000..b03bf87bea --- /dev/null +++ b/src/agent/service/managed-node-broker.ts @@ -0,0 +1,153 @@ +import { decodeBrokerRunId } from "./broker-run-route.ts"; +import { + createVeryfrontServer, + type NodeVeryfrontServiceServer, + startNodeVeryfrontServer, + type VeryfrontServiceServerLogger, +} from "../../server/service-server.ts"; + +/** Trusted broker route handler and optional retirement hook. */ +export interface ManagedNodeBrokerHandler { + handle(request: Request, input: { runId?: string }): Response | Promise; + close?: () => void | Promise; +} + +/** Broker admission and settlement lifecycle retained by the HTTP server. */ +export interface ManagedNodeBrokerPool { + shutdown(): Promise; + readonly closed: Promise; + readonly settled: Promise; +} + +/** Bind managed routes and stop admission before joining all work during shutdown. */ +export async function startNodeManagedAgentBroker(options: { + port: number; + bindAddress?: string; + signals?: readonly NodeJS.Signals[]; + hardShutdownTimeoutMs?: number; + logger?: VeryfrontServiceServerLogger; + broker: ManagedNodeBrokerPool; + readiness(): boolean | Promise; + handlers: { + signedStream: ManagedNodeBrokerHandler; + durableStart: ManagedNodeBrokerHandler; + agUi: ManagedNodeBrokerHandler; + cancel: ManagedNodeBrokerHandler; + resume: ManagedNodeBrokerHandler; + }; +}): Promise { + validateOptions(options); + let shuttingDown = false; + let shutdown: Promise | undefined; + const beginShutdown = () => { + shuttingDown = true; + if (!shutdown) { + shutdown = (async () => await options.broker.shutdown())(); + } + void shutdown.catch(() => {}); + return shutdown; + }; + const handlers = Object.values(options.handlers); + const runtime = createVeryfrontServer({ + logger: options.logger, + modules: [{ + name: "managed-agent-broker", + async handle(request) { + const url = new URL(request.url); + const route = resolveRoute(request.method, url.pathname); + if (route?.kind === "invalid") { + return Response.json({ errorCode: "BROKER_INGRESS_TARGET_MISMATCH" }, { status: 400 }); + } + if (route?.kind === "liveness") return new Response("OK"); + if (route?.kind === "ready") { + const ready = !shuttingDown && await options.readiness(); + return new Response(ready ? "OK" : shuttingDown ? "Shutting down" : "Not Ready", { + status: ready ? 200 : 503, + }); + } + if (!route) return null; + if (shuttingDown) { + return Response.json({ errorCode: "BROKER_UNAVAILABLE" }, { status: 503 }); + } + return await options.handlers[route.kind].handle(request, { + ...("runId" in route ? { runId: route.runId } : {}), + }); + }, + setShuttingDown() { + beginShutdown(); + }, + async stop() { + const results = await Promise.allSettled([ + beginShutdown(), + options.broker.closed, + options.broker.settled, + ...[...new Set(handlers)].map((handler) => + Promise.resolve().then(() => handler.close?.()) + ), + ]); + const failed = results.find((result) => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + }, + }], + }); + const server = await startNodeVeryfrontServer({ + runtime, + port: options.port, + bindAddress: options.bindAddress, + signals: options.signals, + hardShutdownTimeoutMs: options.hardShutdownTimeoutMs, + logger: options.logger, + }); + await server.ready; + return server; +} + +type RunRoute = { kind: "signedStream" | "cancel" | "resume"; runId: string }; + +type Route = + | RunRoute + | { kind: "durableStart" | "agUi" | "liveness" | "ready" | "invalid" }; + +function resolveRoute(method: string, pathname: string): Route | undefined { + if (method === "GET" && pathname === "/liveness") return { kind: "liveness" }; + if (method === "GET" && pathname === "/readiness") return { kind: "ready" }; + if (method === "POST" && pathname === "/api/runs") return { kind: "durableStart" }; + if (method === "POST" && pathname === "/api/ag-ui") return { kind: "agUi" }; + const signed = /^\/api\/control-plane\/runs\/([^/]+)\/stream$/u.exec(pathname); + if (method === "POST" && signed) return decodeRunRoute("signedStream", signed[1]!); + const resume = /^\/api\/runs\/([^/]+)\/resume$/u.exec(pathname); + if (method === "POST" && resume) return decodeRunRoute("resume", resume[1]!); + const controlResume = /^\/api\/control-plane\/runs\/([^/]+)\/resume$/u.exec(pathname); + if (method === "POST" && controlResume) { + return decodeRunRoute("resume", controlResume[1]!); + } + const cancel = /^\/api\/runs\/([^/]+)$/u.exec(pathname); + if (method === "DELETE" && cancel) return decodeRunRoute("cancel", cancel[1]!); + const controlCancel = /^\/api\/control-plane\/runs\/([^/]+)$/u.exec(pathname); + if (method === "DELETE" && controlCancel) { + return decodeRunRoute("cancel", controlCancel[1]!); + } + return undefined; +} + +function decodeRunRoute(kind: RunRoute["kind"], value: string): Route { + const runId = decodeBrokerRunId(value); + return runId === null ? { kind: "invalid" } : { kind, runId }; +} + +function validateOptions(options: { + broker: ManagedNodeBrokerPool; + readiness: unknown; + handlers: Record; +}): void { + if ( + !options.broker || typeof options.broker.shutdown !== "function" || + !(options.broker.closed instanceof Promise) || !(options.broker.settled instanceof Promise) || + typeof options.readiness !== "function" + ) throw new TypeError("Managed broker lifecycle configuration is incomplete"); + for (const name of ["signedStream", "durableStart", "agUi", "cancel", "resume"]) { + if (typeof options.handlers?.[name]?.handle !== "function") { + throw new TypeError("Managed broker route configuration is incomplete"); + } + } +} diff --git a/tests/e2e/agent/managed-broker/executor.mjs b/tests/e2e/agent/managed-broker/executor.mjs new file mode 100644 index 0000000000..b5a3f08408 --- /dev/null +++ b/tests/e2e/agent/managed-broker/executor.mjs @@ -0,0 +1,28 @@ +import { readFileSync } from "node:fs"; +import process from "node:process"; +import { + initializeExecutorRuntimeContracts, + startExecutorRuntimeEntrypoint, +} from "veryfront/agent/executor-runtime"; + +await initializeExecutorRuntimeContracts(); +const executor = await startExecutorRuntimeEntrypoint({ + readKey: () => Promise.resolve(new Uint8Array(readFileSync(0))), + readArtifact: () => + Promise.resolve({ + manifest: { + version: 1, + root: "project", + owner: { scopeKind: "global", serviceName: "synthetic-broker" }, + source: { type: "release", releaseId: "synthetic-release" }, + }, + projectDir: process.argv[2], + }), +}); +process.stdout.write(`${JSON.stringify({ pid: process.pid, port: executor.address.port })}\n`); +try { + const channel = await executor.ready; + await channel.closed; +} finally { + await executor.close(); +} diff --git a/tests/e2e/agent/managed-broker/journey.mjs b/tests/e2e/agent/managed-broker/journey.mjs new file mode 100644 index 0000000000..333b31e910 --- /dev/null +++ b/tests/e2e/agent/managed-broker/journey.mjs @@ -0,0 +1,659 @@ +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { createHash, generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; +import { assert, assertEquals } from "veryfront/testing/assert"; +import { it } from "veryfront/testing/bdd"; +import { initializeExecutorRuntimeContracts } from "veryfront/agent/executor-runtime"; +import { + connectExecutorTransport, + createManagedBrokerHandler, + createManagedBrokerPersistence, + createManagedExecutorBroker, + startNodeManagedAgentBroker, +} from "veryfront/agent/managed-broker"; + +await initializeExecutorRuntimeContracts(); +const modelId = "veryfront-cloud/openai/synthetic"; +const owner = { scopeKind: "global", serviceName: "synthetic-broker" }; +const source = { type: "release", releaseId: "synthetic-release" }; +const projectId = "00000000-0000-4000-8000-000000000005"; +const conversationId = "00000000-0000-4000-8000-000000000001"; +const messageId = "00000000-0000-4000-8000-000000000002"; +const image = `registry.example.test/executor@sha256:${"a".repeat(64)}`; +const usage = { inputTokens: 4, outputTokens: 2, totalTokens: 6 }; +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +async function bounded(promise, label, ms = 25_000) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out`)), ms); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +async function scenario(kind) { + const steering = kind === "steering"; + const providerToolNames = steering ? ["web_search"] : []; + const mode = kind === "sse" || kind === "disconnect" ? "sse" : "detached"; + const project = new URL(`./project-${kind}/`, import.meta.url); + await mkdir(new URL("agents/", project), { recursive: true }); + await copyFile(new URL("./project-hooks.mjs", import.meta.url), new URL("hooks.mjs", project)); + await writeFile( + new URL("veryfront.config.ts", project), + 'import "./hooks.mjs"; export default { ai: { agents: { discovery: { paths: ["agents"] } } } };', + ); + await writeFile( + new URL("agents/probe.ts", project), + `import { agent } from "veryfront/agent"; +export default agent({ id: "probe", model: ${JSON.stringify(modelId)}, + system: "Use host_probe once, then report its result.", + tools: ${ + JSON.stringify(steering ? { host_probe: true, update_file: true } : { host_probe: true }) + }, + providerTools: ${JSON.stringify(providerToolNames)} });`, + ); + + const secrets = Object.fromEntries(["authorization", "api", "inference", "events"].map( + (name) => [name, `synthetic-broker-private-${name}-${randomUUID()}`], + )); + const canaries = Object.values(secrets); + const runId = `run-${kind}`; + const path = `/api/control-plane/runs/${runId}/stream`; + const run = { + runId, + conversationId, + messageId, + latestEventId: 0, + latestExternalEventSequence: 0, + waitingToolCallId: null, + waitingToolName: null, + status: "running", + streamProtocolVersion: 2, + }; + const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + const body = JSON.stringify({ + run: { + agentServiceId: "synthetic-service", + agentId: "probe", + conversationId, + runId, + messageId, + inputAnchorMessageId: "00000000-0000-4000-8000-000000000003", + requestedByUserId: "00000000-0000-4000-8000-000000000006", + project: { projectId, projectSlug: "demo-project", runtimeTargetKind: "main_branch" }, + }, + messages: [{ id: "user", role: "user", content: "Run the host probe." }], + tools: [], + context: [], + agentSource: source, + credentials: { authToken: secrets.api, inferenceAuthToken: secrets.inference }, + }); + const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const nowSeconds = Math.floor(Date.now() / 1000); + const signed = `${encode({ alg: "EdDSA", typ: "JWT" })}.${ + encode({ + iss: "veryfront-api", + aud: "demo-project", + sub: runId, + surface: "studio", + project_id: projectId, + request_hash: createHash("sha256").update(body).digest("base64url"), + request_method: "POST", + request_path: path, + iat: nowSeconds, + exp: nowSeconds + 90, + }) + }`; + const headers = { + authorization: `Bearer ${secrets.authorization}`, + "x-veryfront-control-plane-jws": `${signed}.${ + sign(null, Buffer.from(signed), privateKey).toString("base64url") + }`, + "x-veryfront-run-event-token": secrets.events, + "content-type": "application/json", + }; + const clientAbort = new AbortController(); + const executionAbort = new AbortController(); + const secondModelCall = Promise.withResolvers(); + const terminalEntered = Promise.withResolvers(); + const terminalRelease = Promise.withResolvers(); + const finished = Promise.withResolvers(); + const allocations = []; + const releases = []; + const persisted = []; + const completions = []; + const modelCalls = []; + const steeringRefreshes = []; + const tools = []; + const apiErrors = []; + let cursor = 0; + let child; + let childExited; + let childOutput = ""; + let handler; + let server; + let shutdown; + let transportClosed = false; + const broker = createManagedExecutorBroker({ maxActive: 1 }); + + // A real HTTP endpoint checks the persistence wire contract. No live API or + // provider is contacted; credentials are freshly generated synthetic canaries. + const api = createServer(async (request, response) => { + try { + assertEquals(request.headers.authorization, `Bearer ${secrets.events}`); + let raw = ""; + for await (const chunk of request) raw += chunk; + const data = JSON.parse(raw); + for (const canary of canaries) { + assert(!raw.includes(canary), "Credential entered persistence data"); + } + response.setHeader("content-type", "application/json"); + if (Array.isArray(data.events)) { + assertEquals(request.url, `/conversations/${conversationId}/runs/${runId}/events`); + persisted.push(...data.events); + cursor += data.events.length; + response.end(JSON.stringify({ + latest_event_id: cursor, + latest_external_event_sequence: cursor, + appended_count: data.events.length, + run: { + run_id: runId, + conversation_id: conversationId, + latest_event_id: cursor, + latest_external_event_sequence: cursor, + }, + })); + } else { + assertEquals(request.url, `/runs/${runId}/complete`); + completions.push(data); + terminalEntered.resolve(); + if (kind === "delayed-persistence") await terminalRelease.promise; + response.end(JSON.stringify({ completed: true, run: { runId, status: data.status } })); + } + } catch (error) { + apiErrors.push(error); + response.statusCode = 500; + response.end("{}"); + } + }); + await new Promise((resolve) => api.listen(0, "127.0.0.1", resolve)); + const apiUrl = `http://127.0.0.1:${api.address().port}`; + + const model = { + specificationVersion: "v3", + provider: "openai", + modelId: "synthetic", + doGenerate() { + throw new Error("Unexpected non-streaming model call"); + }, + doStream(options) { + modelCalls.push(options); + const turn = modelCalls.length; + for (const canary of canaries) { + assert( + !JSON.stringify(options).includes(canary), + "Credential entered model application data", + ); + } + assert(turn <= 2, "Unexpected extra model call"); + return Promise.resolve({ + stream: new ReadableStream({ + start(controller) { + if (turn === 1) { + controller.enqueue({ + type: "tool-call", + toolCallId: "host-call", + toolName: "host_probe", + input: {}, + }); + if (steering) { + controller.enqueue({ + type: "tool-call", + toolCallId: "steering-call", + toolName: "update_file", + input: { path: "AGENTS.md", project_reference: projectId }, + }); + } + controller.enqueue({ type: "finish", finishReason: "tool-calls", totalUsage: usage }); + controller.close(); + } else { + controller.enqueue({ type: "text-delta", text: "Host tool completed." }); + secondModelCall.resolve(); + if (kind === "kill" || kind === "disconnect") { + const abort = () => controller.error(new Error("Synthetic provider cancelled")); + if (options.abortSignal.aborted) abort(); + else options.abortSignal.addEventListener("abort", abort, { once: true }); + } else { + controller.enqueue({ type: "finish", finishReason: "stop", totalUsage: usage }); + controller.close(); + } + } + }, + }), + }); + }, + }; + + try { + handler = createManagedBrokerHandler({ + broker, + responseMode: mode, + resolveIngressOptions: () => ({ + publicKeyPem: publicKey.export({ type: "spki", format: "pem" }), + audience: "demo-project", + projectId, + expectedSurface: "studio", + boundSource: source, + expectedOwner: owner, + authorizeScope(authority) { + assertEquals(authority.authorization, headers.authorization); + assertEquals(authority.apiAuthToken, secrets.api); + assertEquals(authority.runEventToken, secrets.events); + return { authorized: true }; + }, + }), + prepare({ ingress }) { + assertEquals(ingress.privateAuthority.inferenceAuthToken, secrets.inference); + const persistence = createManagedBrokerPersistence({ + apiUrl, + runEventToken: ingress.privateAuthority.runEventToken, + run, + modelId, + resolveProvider: () => "openai", + fetch: globalThis.fetch, + }); + const now = Date.now(); + const allocationRequest = { + allocationId: randomUUID(), + invocationId: randomUUID(), + owner, + source, + requestedAt: now, + prepareDeadlineAt: now + 45_000, + hardDeadlineAt: now + 90_000, + }; + const binding = { + ...allocationRequest, + generation: 1, + brokerInstanceId: "synthetic-broker", + }; + delete binding.requestedAt; + delete binding.prepareDeadlineAt; + delete binding.hardDeadlineAt; + const allocation = (phase, reason) => ({ + binding, + phase, + expiresAt: allocationRequest.hardDeadlineAt, + ...(reason ? { reason } : {}), + ...(phase === "ready" + ? { + endpoint: { + address: "127.0.0.1", + port: 8081, + podUid: "synthetic-executor", + nodeName: "synthetic-node", + image, + channelAuthenticated: false, + }, + } + : {}), + }); + const allocator = { + async allocate(request, bootstrap) { + allocations.push(request); + const key = Buffer.from(bootstrap.channelKey); + child = spawn(process.execPath, [ + fileURLToPath(new URL("./executor.mjs", import.meta.url)), + fileURLToPath(project), + ], { + cwd: fileURLToPath(project), + stdio: ["pipe", "pipe", "pipe"], + // Deliberately omit all broker credentials and ambient provider configuration. + env: { + PATH: process.env.PATH, + NODE_ENV: "production", + VF_DISABLE_LRU_INTERVAL: "1", + VERYFRONT_EXECUTOR_ALLOCATION_ID: request.allocationId, + VERYFRONT_EXECUTOR_INVOCATION_ID: request.invocationId, + VERYFRONT_EXECUTOR_GENERATION: "1", + VERYFRONT_EXECUTOR_ACTIVE_DEADLINE_SECONDS: "90", + VERYFRONT_EXECUTOR_HARD_DEADLINE_AT: String(request.hardDeadlineAt), + PORT: "8081", + }, + }); + childExited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }); + void childExited.catch(() => {}); + const ready = Promise.withResolvers(); + let stdout = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + childOutput += chunk; + if (stdout.includes("\n")) { + try { + ready.resolve(JSON.parse(stdout.split("\n")[0])); + } catch { + ready.reject(new Error("Invalid executor readiness")); + } + } + }); + child.stderr.on("data", (chunk) => { + childOutput += chunk; + }); + void childExited.then( + () => ready.reject(new Error("Executor exited before readiness")), + ready.reject, + ); + child.stdin.end(key, () => key.fill(0)); + const endpoint = await bounded(ready.promise, "Executor readiness"); + assert(endpoint.pid !== process.pid); + assertEquals(endpoint.port, 8081); + return allocation("ready"); + }, + observe: () => Promise.resolve(allocation("ready")), + renew: () => Promise.resolve(allocation("ready")), + release(_binding, reason) { + releases.push(reason); + return Promise.resolve(allocation("released", reason)); + }, + }; + return Promise.resolve({ + start: { + bindSessionOwnedWork: persistence.bindSessionOwnedWork, + session: { + request: allocationRequest, + expectedBrokerInstanceId: "synthetic-broker", + expectedImage: image, + allocator, + requestTimeoutMs: 30_000, + cleanupTimeoutMs: 200, + async connectTransport(options) { + const transport = await connectExecutorTransport(options); + return { + ...transport, + close() { + transportClosed = true; + return transport.close(); + }, + }; + }, + }, + installation: { + version: 1, + root: "project", + owner, + source, + grant: { + agentId: "probe", + defaultModelId: modelId, + maxSteps: 4, + models: [{ id: modelId, maxOutputTokens: 100, providerToolNames }], + allowedToolNames: steering ? ["host_probe", "update_file"] : ["host_probe"], + hostToolFacadeIds: ["host"], + remoteToolSourceIds: steering ? ["state-tools"] : [], + execution: { + kind: "canonical", + projectId: steering ? projectId : null, + conversationId, + runId, + messageId, + providerReplay: "disabled", + }, + }, + capabilities: { + persistence: { publishParentRunEvents: "parent", toolExposureCheckpoint: "tools" }, + ...(steering ? { projectSteering: "steering" } : {}), + }, + }, + prepare: { agentId: "probe" }, + model: { + resolver: () => model, + runEventSink: persistence.modelRunEventSink, + grant: { + maxCalls: 3, + maxConcurrentCalls: 1, + models: new Map([[modelId, { + maxOutputTokens: 100, + providerTools: providerToolNames.map((name) => ({ + type: "provider", + name, + id: `openai.${name}`, + args: {}, + })), + }]]), + }, + }, + tools: { + catalog: new Map([ + ["host_probe", {}], + ...(steering ? [["update_file", {}]] : []), + ]), + maxCalls: 8, + maxConcurrent: 1, + sources: new Map([ + ["host", { + allowedToolNames: new Set(["host_probe"]), + context: {}, + source: { + id: "host", + listTools: () => + Promise.resolve([{ + name: "host_probe", + description: "Read a synthetic result", + parameters: { type: "object", properties: {}, additionalProperties: false }, + }]), + executeTool(name) { + tools.push(name); + return Promise.resolve({ text: "host-ok" }); + }, + }, + }], + ...(steering + ? [["state-tools", { + allowedToolNames: new Set(["update_file"]), + context: {}, + source: { + id: "state-tools", + listTools: () => + Promise.resolve([{ + name: "update_file", + description: "Update synthetic project instructions", + parameters: { + type: "object", + properties: { + path: { type: "string" }, + project_reference: { type: "string" }, + }, + required: ["path", "project_reference"], + }, + }]), + executeTool(name) { + tools.push(name); + return Promise.resolve({ success: true }); + }, + }, + }]] + : []), + ]), + }, + persistence: { + publishParentRunEvents: persistence.publishParentRunEvents, + persistToolExposureCheckpoint: persistence.persistToolExposureCheckpoint, + initialProviderReplayCheckpoints: [], + }, + state: steering + ? { + prepareProjectSteering: ({ definition }) => Promise.resolve({ agent: definition }), + refreshProjectSteering(_signal, names) { + steeringRefreshes.push( + [...names].sort((left, right) => left.localeCompare(right)), + ); + return Promise.resolve("Synthetic refreshed steering"); + }, + } + : {}, + }, + messages: ingress.executor.input.messages.map((message) => ({ + id: message.id, + role: message.role, + timestamp: 1, + parts: [{ type: "text", text: message.content }], + })), + executionSignal: executionAbort.signal, + output: persistence.output, + async cleanup() { + await persistence.cleanup(); + finished.resolve(); + }, + }); + }, + }); + const unsupported = { handle: () => new Response(null, { status: 404 }) }; + server = await startNodeManagedAgentBroker({ + port: 0, + bindAddress: "127.0.0.1", + signals: [], + hardShutdownTimeoutMs: 10_000, + broker, + readiness: () => true, + handlers: { + signedStream: handler, + durableStart: unsupported, + agUi: unsupported, + cancel: unsupported, + resume: unsupported, + }, + }); + if (kind === "detached") { + const rejected = await fetch(`${server.url}${path}`, { + method: "POST", + body, + headers: { ...headers, "x-veryfront-control-plane-jws": "invalid" }, + }); + assertEquals(rejected.status, 401); + await rejected.body?.cancel(); + assertEquals(allocations.length, 0, "Invalid signatures must not allocate an executor"); + } + const response = await fetch(`${server.url}${path}`, { + method: "POST", + headers, + body, + signal: clientAbort.signal, + }); + assertEquals( + response.status, + mode === "sse" ? 200 : 202, + await (response.status >= 400 ? response.text() : Promise.resolve("")), + ); + let wire = ""; + if (mode === "detached") { + assertEquals(await response.json(), { accepted: true, duplicate: false }); + } + const reading = mode === "sse" + ? response.text().then((text) => { + wire = text; + }) + : Promise.resolve(); + void reading.catch(() => {}); + await bounded(secondModelCall.promise, "Second model call"); + + if (kind === "kill") { + const duplicate = await fetch(`${server.url}${path}`, { method: "POST", headers, body }); + assertEquals(await duplicate.json(), { accepted: true, duplicate: true }); + child.kill("SIGKILL"); + } else if (kind === "disconnect") { + clientAbort.abort(); + await reading.catch(() => {}); + } else if (kind === "delayed-persistence") { + await bounded(terminalEntered.promise, "Terminal persistence"); + shutdown = server.stop(); + await bounded(broker.closed, "Bounded broker closure"); + assertEquals(broker.active, 1, "Pending persistence must retain admission"); + let settled = false; + void broker.settled.then(() => { + settled = true; + }); + await tick(); + assertEquals(settled, false); + terminalRelease.resolve(); + } + await bounded(finished.promise, "Run cleanup"); + if (kind !== "disconnect") await bounded(reading, "SSE completion"); + const exit = await bounded(childExited, "Executor retirement"); + assertEquals(allocations.length, 1); + assertEquals(releases.length, 1); + assertEquals(transportClosed, true); + assertEquals( + [...tools].sort((left, right) => left.localeCompare(right)), + steering ? ["host_probe", "update_file"] : ["host_probe"], + ); + assertEquals(modelCalls.length, 2); + assert(JSON.stringify(modelCalls[0].prompt).includes("Run the host probe.")); + assert(JSON.stringify(modelCalls[1].prompt).includes("host-ok")); + if (steering) { + assertEquals(steeringRefreshes, [["host_probe", "update_file", "web_search"]]); + for (const call of modelCalls) { + assert(call.tools.some((tool) => tool.name === "web_search")); + } + assert(JSON.stringify(modelCalls[1].prompt).includes("Synthetic refreshed steering")); + } + assertEquals(apiErrors, []); + if (kind === "kill") assertEquals(exit.signal, "SIGKILL"); + else assertEquals(exit.code, 0); + if (mode === "detached") { + assertEquals(completions.length, 1, "Exactly one durable terminal update"); + assertEquals(completions[0].status, kind === "kill" ? "failed" : "completed"); + if (kind !== "kill") assert(JSON.stringify(persisted).includes("Host tool completed.")); + } else { + assertEquals(completions.length, 0, "SSE persistence is owned by the stream consumer"); + if (kind === "sse") { + assertEquals(wire.split("\n").filter((line) => line === "event: RunFinished").length, 1); + assert(wire.includes("Host tool completed.")); + assert(!wire.includes("event: RunError")); + } + } + assert(persisted.length > 0, "Canonical model/tool events must reach persistence"); + const observation = JSON.parse(await readFile(new URL("observations.json", project), "utf8")); + assertEquals(observation.pid, child.pid); + assertEquals(observation.controls, { + call: true, + json: true, + decode: true, + headers: true, + tee: true, + }); + assertEquals(observation.observations, 0, "Project hooks observed a broker canary"); + for (const canary of canaries) { + assert(!`${wire}${childOutput}`.includes(canary), "Credential entered executor output"); + } + await bounded(shutdown ?? server.stop(), "Server shutdown"); + await bounded(broker.settled, "Broker settlement"); + assertEquals(broker.active, 0); + } finally { + terminalRelease.resolve(); + clientAbort.abort(); + executionAbort.abort(); + child?.kill("SIGKILL"); + await childExited?.catch(() => {}); + await bounded(shutdown ?? server?.stop() ?? broker.shutdown(), "Cleanup").catch(() => {}); + api.closeAllConnections(); + await new Promise((resolve) => api.close(resolve)); + await rm(project, { recursive: true, force: true }); + } +} + +for (const kind of ["sse", "detached", "kill", "disconnect", "delayed-persistence", "steering"]) { + it(`packed managed broker: ${kind}`, { timeout: 90_000 }, () => scenario(kind)); +} diff --git a/tests/e2e/agent/managed-broker/project-hooks.mjs b/tests/e2e/agent/managed-broker/project-hooks.mjs new file mode 100644 index 0000000000..ad2ff7854d --- /dev/null +++ b/tests/e2e/agent/managed-broker/project-hooks.mjs @@ -0,0 +1,69 @@ +import { writeFileSync } from "node:fs"; +import process from "node:process"; + +// Only synthetic marker prefixes are known to the project. The actual broker +// canaries are random and never included in its environment or source files. +const prefix = "synthetic-broker-private-"; +const control = `${prefix}positive-control`; +const stringify = JSON.stringify; +const apply = Reflect.apply; +const includes = String.prototype.includes; +const report = { pid: process.pid, controls: {}, observations: 0 }; +const reportPath = new URL("./observations.json", import.meta.url); +function record(kind, value) { + if (typeof value !== "string" || !apply(includes, value, [prefix])) return; + if (apply(includes, value, [control])) report.controls[kind] = true; + else report.observations++; + writeFileSync(reportPath, stringify(report)); +} + +Function.prototype.call = new Proxy(Function.prototype.call, { + apply(target, receiver, args) { + for (let index = 0; index < args.length; index++) record("call", args[index]); + return apply(target, receiver, args); + }, +}); +JSON.stringify = new Proxy(JSON.stringify, { + apply(target, receiver, args) { + const value = apply(target, receiver, args); + record("json", value); + return value; + }, +}); +TextDecoder.prototype.decode = new Proxy(TextDecoder.prototype.decode, { + apply(target, receiver, args) { + const value = apply(target, receiver, args); + record("decode", value); + return value; + }, +}); +const iterator = Object.getPrototypeOf(new Headers().entries()); +iterator.next = new Proxy(iterator.next, { + apply(target, receiver, args) { + const next = apply(target, receiver, args); + if (!next.done) record("headers", next.value[1]); + return next; + }, +}); + +const originalTee = ReadableStream.prototype.tee; +const pendingReads = []; +ReadableStream.prototype.tee = function () { + const branches = apply(originalTee, this, []); + const [forward, observer] = apply(originalTee, branches[0], []); + pendingReads.push(new Response(observer).text().then((value) => record("tee", value))); + return [forward, branches[1]]; +}; + +// Positive controls prevent a passing probe with inactive hooks. +(function () {}).call(null, control); +JSON.stringify({ value: control }); +new TextDecoder().decode(new TextEncoder().encode(control)); +new Headers({ "x-synthetic": control }).entries().next(); +record("environment", stringify(process.env)); +writeFileSync(reportPath, stringify(report)); + +const bodyControl = new Response(control).body; +const controlBranches = bodyControl.tee(); +await Promise.all(controlBranches.map((branch) => new Response(branch).text())); +await Promise.all(pendingReads); diff --git a/tests/fixtures/executor-runtime-process.ts b/tests/fixtures/executor-runtime-process.ts new file mode 100644 index 0000000000..66deb496d1 --- /dev/null +++ b/tests/fixtures/executor-runtime-process.ts @@ -0,0 +1,33 @@ +import { readFileSync } from "node:fs"; +import process from "node:process"; +import { + initializeExecutorRuntimeContracts, + startExecutorRuntimeEntrypoint, +} from "#veryfront/agent/hosted/executor-runtime-entrypoint.ts"; + +await initializeExecutorRuntimeContracts(); + +// Synthetic allocation key arrives on a private pipe; no broker environment or +// HTTP data is inherited by this executor process. +const executor = await startExecutorRuntimeEntrypoint({ + readKey: () => Promise.resolve(new Uint8Array(readFileSync(0))), + readArtifact: () => + Promise.resolve({ + manifest: { + version: 1, + root: "project", + owner: { scopeKind: "global", serviceName: "veryfront-agent" }, + source: { type: "release", releaseId: "synthetic-release" }, + }, + projectDir: process.argv[2]!, + }), +}); +process.stdout.write( + `${JSON.stringify({ ready: true, pid: process.pid, port: executor.address.port })}\n`, +); +try { + const channel = await executor.ready; + await channel.closed; +} finally { + await executor.close(); +} diff --git a/tests/integration/agent/executor-allocator-client.test.ts b/tests/integration/agent/executor-allocator-client.test.ts index 0ca931a235..d775daa26e 100644 --- a/tests/integration/agent/executor-allocator-client.test.ts +++ b/tests/integration/agent/executor-allocator-client.test.ts @@ -123,6 +123,23 @@ if ("Deno" in globalThis || "Bun" in globalThis) { } else { const tls = certificate(); describe("executor allocator HTTPS client", () => { + it("accepts native Node with a Deno compatibility namespace", () => { + const original = Object.getOwnPropertyDescriptor(globalThis, "Deno"); + Object.defineProperty(globalThis, "Deno", { + configurable: true, + value: { version: { deno: "compatibility" } }, + }); + try { + const client = createHostedExecutorAllocatorClient({ + baseUrl: "https://allocator.example.test", + readBrokerToken: () => Promise.resolve("synthetic-token"), + }); + assertEquals(typeof client.allocate, "function"); + } finally { + if (original) Object.defineProperty(globalThis, "Deno", original); + else Reflect.deleteProperty(globalThis, "Deno"); + } + }); it("retains allocator DNS work until the native lookup settles", async () => { if (process.env.VF_EXECUTOR_DNS_RETIREMENT_TEST === "1") { let nativeWorkDone = false; diff --git a/tests/integration/agent/executor-node-bootstrap.test.ts b/tests/integration/agent/executor-node-bootstrap.test.ts index 6bbba33b7c..bd62081c9a 100644 --- a/tests/integration/agent/executor-node-bootstrap.test.ts +++ b/tests/integration/agent/executor-node-bootstrap.test.ts @@ -17,6 +17,7 @@ import { connectExecutorTransport } from "#veryfront/agent/hosted/executor-node- import { register, tryResolve, unregister } from "#veryfront/extensions/contracts.ts"; import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; +import { registerExecutorRuntimeEntrypointTests } from "./executor-runtime-entrypoint.fixture.ts"; const binding = { allocationId: "00000000-0000-4000-8000-000000000001", @@ -84,6 +85,34 @@ if (typeof Deno !== "undefined") { ); } else { describe("fixed Node executor bootstrap", () => { + registerExecutorRuntimeEntrypointTests(); + it("accepts native Node when a Deno compatibility namespace is present", async () => { + const original = Object.getOwnPropertyDescriptor(globalThis, "Deno"); + const key = randomBytes(32); + let bootstrap: Awaited> | undefined; + let caller: Awaited> | undefined; + Object.defineProperty(globalThis, "Deno", { + configurable: true, + value: { version: { deno: "compatibility" } }, + }); + try { + bootstrap = await startExecutorNodeBootstrap({ + operations, + environment: environment(), + readKey: () => Promise.resolve(new Uint8Array(key)), + }); + caller = await connectCaller(bootstrap.address.port, key); + await caller.ready; + assertEquals(await caller.request("echo", "packaged-node"), "packaged-node"); + } finally { + if (original) Object.defineProperty(globalThis, "Deno", original); + else Reflect.deleteProperty(globalThis, "Deno"); + caller?.close(); + bootstrap?.close(); + await caller?.settled; + await bootstrap?.ready.then((channel) => channel.settled, () => {}); + } + }); it("rejects missing and noncanonical bootstrap values before reading a key", async () => { let reads = 0; const invalid: Record[] = Object.keys(values).map((name) => ({ diff --git a/tests/integration/agent/executor-runtime-contracts.test.ts b/tests/integration/agent/executor-runtime-contracts.test.ts new file mode 100644 index 0000000000..1de4ad3b84 --- /dev/null +++ b/tests/integration/agent/executor-runtime-contracts.test.ts @@ -0,0 +1,70 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assert, assertEquals } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; +import { tryResolve } from "#veryfront/extensions/contracts.ts"; +import { initializeExecutorRuntimeContracts } from "#veryfront/agent/hosted/executor-runtime-contracts.ts"; + +it("initializes executor runtime contracts once and preserves their trusted generations", async () => { + await Promise.all([ + initializeExecutorRuntimeContracts(), + initializeExecutorRuntimeContracts(), + ]); + const first = [ + tryResolve("SchemaValidator"), + tryResolve("Bundler"), + tryResolve("ModuleLexer"), + tryResolve("SkillDocumentParserProvider"), + ]; + assertEquals(first.every((contract) => contract !== undefined), true); + await initializeExecutorRuntimeContracts(); + assertEquals([ + tryResolve("SchemaValidator"), + tryResolve("Bundler"), + tryResolve("ModuleLexer"), + tryResolve("SkillDocumentParserProvider"), + ], first); +}); + +it("keeps project runtime modules outside the contract initializer graph", async () => { + const root = new URL("../../../", import.meta.url); + const command = new Deno.Command(Deno.execPath(), { + cwd: root, + args: [ + "info", + "--frozen", + "--json", + "src/agent/hosted/executor-runtime-contracts.ts", + ], + }); + const output = await command.output(); + assertEquals(output.code, 0, new TextDecoder().decode(output.stderr)); + const graph = JSON.parse(new TextDecoder().decode(output.stdout)) as { + roots: string[]; + modules: Array<{ specifier: string; dependencies?: Array<{ code?: { specifier: string } }> }>; + }; + const modules = new Map(graph.modules.map((module) => [module.specifier, module])); + const visited = new Set(); + const pending = [...graph.roots]; + while (pending.length) { + const specifier = pending.shift()!; + if (visited.has(specifier)) continue; + visited.add(specifier); + for (const dependency of modules.get(specifier)?.dependencies ?? []) { + if (dependency.code) pending.push(dependency.code.specifier); + } + } + const forbidden = [ + "/src/config/loader.ts", + "/src/agent/project/agent-runtime.ts", + "/src/agent/hosted/cloud-agent-config.ts", + "/src/agent/hosted/executor-discovery-node.ts", + "/src/agent/hosted/default-chat-runtime.ts", + ]; + assertEquals( + [...visited].filter((specifier) => forbidden.some((path) => specifier.endsWith(path))), + [], + ); + assert( + [...visited].some((specifier) => specifier.endsWith("/src/extensions/bundler/defaults.ts")), + ); +}); diff --git a/tests/integration/agent/executor-runtime-entrypoint.fixture.ts b/tests/integration/agent/executor-runtime-entrypoint.fixture.ts new file mode 100644 index 0000000000..8ac94c8923 --- /dev/null +++ b/tests/integration/agent/executor-runtime-entrypoint.fixture.ts @@ -0,0 +1,199 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { spawn } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { assert, assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; +import { createExecutorChannel } from "#veryfront/agent/executor/channel.ts"; +import { connectExecutorTransport } from "#veryfront/agent/hosted/executor-node-transport.ts"; +import { createExecutorModelBroker } from "#veryfront/agent/hosted/executor-model-bridge.ts"; +import { getExecutorDiscoveryResultSchema } from "#veryfront/agent/hosted/executor-discovery-schema.ts"; +import { startExecutorRuntimeEntrypoint } from "#veryfront/agent/hosted/executor-runtime-entrypoint.ts"; + +const root = new URL("../../../", import.meta.url); +const resolver = fileURLToPath(new URL("tests/node/resolver.mjs", root)); + +/** Register beside the fixed-port bootstrap tests so Deno file parallelism cannot race port 8081. */ +export function registerExecutorRuntimeEntrypointTests(): void { + it("rejects missing first-party runtime contracts before reading an allocation key", async () => { + let keyReads = 0; + let executor: Awaited> | undefined; + const values: Record = { + VERYFRONT_EXECUTOR_ALLOCATION_ID: randomUUID(), + VERYFRONT_EXECUTOR_INVOCATION_ID: randomUUID(), + VERYFRONT_EXECUTOR_GENERATION: "1", + VERYFRONT_EXECUTOR_ACTIVE_DEADLINE_SECONDS: "30", + VERYFRONT_EXECUTOR_HARD_DEADLINE_AT: String(Date.now() + 30_000), + PORT: "8081", + }; + try { + await assertRejects(async () => { + executor = await startExecutorRuntimeEntrypoint({ + environment: { get: (name) => values[name] }, + readArtifact: () => + Promise.resolve({ + manifest: { + version: 1, + root: "project", + owner: { scopeKind: "global", serviceName: "veryfront-agent" }, + source: { type: "release", releaseId: "synthetic-release" }, + }, + projectDir: "/synthetic-project", + }), + readKey: () => { + keyReads++; + return Promise.resolve(randomBytes(32)); + }, + }); + }); + assertEquals(keyReads, 0); + } finally { + await executor?.close(); + } + }); + it("loads a real project only in the executor after the fixed installation operation", { + timeout: 45_000, + }, async () => { + const dir = await mkdtemp(join(tmpdir(), "vf-managed-executor-")); + const marker = join(dir, "loaded.json"); + await mkdir(join(dir, "crew")); + await writeFile( + join(dir, "veryfront.config.ts"), + `import { writeFileSync } from "node:fs"; import process from "node:process"; writeFileSync(${ + JSON.stringify(marker) + }, JSON.stringify({pid:process.pid})); export default { ai: { agents: { discovery: { paths: ["crew"] } } } };`, + ); + await writeFile( + join(dir, "crew", "writer.md"), + "---\nname: Writer\n---\nSynthetic instructions.\n", + ); + const binding = { allocationId: randomUUID(), generation: 1, invocationId: randomUUID() }; + const key = randomBytes(32); + const child = spawn(process.execPath, [ + "--import", + resolver, + fileURLToPath(new URL("tests/fixtures/executor-runtime-process.ts", root)), + dir, + ], { + cwd: fileURLToPath(root), + env: { + PATH: "/usr/local/bin:/usr/bin:/bin", + VERYFRONT_EXECUTOR_ALLOCATION_ID: binding.allocationId, + VERYFRONT_EXECUTOR_INVOCATION_ID: binding.invocationId, + VERYFRONT_EXECUTOR_GENERATION: "1", + VERYFRONT_EXECUTOR_ACTIVE_DEADLINE_SECONDS: "40", + VERYFRONT_EXECUTOR_HARD_DEADLINE_AT: String(Date.now() + 40_000), + PORT: "8081", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => stderr += chunk); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", resolve); + }); + void exited.catch(() => {}); + const ready = new Promise<{ pid: number; port: number }>((resolve, reject) => { + let output = ""; + child.stdout.on("data", (chunk) => { + output += chunk; + if (output.includes("\n")) { + try { + resolve(JSON.parse(output.split("\n")[0]!)); + } catch { + reject(new Error("Invalid executor readiness")); + } + } + }); + void exited.then( + () => reject(new Error(`Executor exited before readiness: ${stderr}`)), + reject, + ); + }); + child.stdin.end(key); + let channel: ReturnType | undefined; + const timer = setTimeout(() => child.kill(), 40_000); + try { + const endpoint = await ready; + assert(endpoint.pid !== process.pid); + assertEquals(existsSync(marker), false); + const transport = await connectExecutorTransport({ + podIp: "127.0.0.1", + port: endpoint.port, + key, + binding, + timeoutMs: 30_000, + }); + const modelId = "veryfront-cloud/openai/synthetic-model"; + channel = createExecutorChannel({ + binding, + transport, + operations: createExecutorModelBroker({ + allowedModelIds: new Set([modelId]), + resolveModelRuntime: () => ({ + provider: "openai", + modelId: "synthetic-model", + specificationVersion: "v3", + doGenerate: () => { + throw new Error("Unexpected model call"); + }, + doStream: () => { + throw new Error("Unexpected model call"); + }, + }), + }), + }); + await channel.ready; + await assertRejects(() => channel!.request("discovery.describe", {})); + assertEquals(existsSync(marker), false); + const input = { + version: 1, + binding, + root: "project", + owner: { scopeKind: "global", serviceName: "veryfront-agent" }, + source: { type: "release", releaseId: "synthetic-release" }, + grant: { + agentId: "writer", + defaultModelId: modelId, + maxSteps: 3, + models: [{ id: modelId, maxOutputTokens: 100, providerToolNames: [] }], + allowedToolNames: [], + hostToolFacadeIds: [], + remoteToolSourceIds: [], + execution: { kind: "ephemeral", projectId: null }, + }, + capabilities: { persistence: {} }, + }; + await assertRejects(() => + channel!.request("runtime.install", { + ...input, + source: { type: "release", releaseId: "wrong" }, + }) + ); + assertEquals(existsSync(marker), false); + assertEquals(await channel.request("runtime.install", input), { installed: true }); + const description = getExecutorDiscoveryResultSchema().parse( + await channel.request("discovery.describe", {}, { timeoutMs: 30_000 }), + ); + assert(description.ok, JSON.stringify(description)); + assertEquals(JSON.parse(await readFile(marker, "utf8")).pid, endpoint.pid); + await assertRejects(() => channel!.request("runtime.install", input)); + channel.close(); + await channel.settled; + assertEquals(await exited, 0, stderr); + } finally { + clearTimeout(timer); + channel?.close(); + await channel?.settled; + child.kill(); + await exited.catch(() => {}); + await rm(dir, { recursive: true, force: true }); + } + }); +} diff --git a/tests/integration/agent/managed-broker-imports.test.ts b/tests/integration/agent/managed-broker-imports.test.ts new file mode 100644 index 0000000000..2c73e500b1 --- /dev/null +++ b/tests/integration/agent/managed-broker-imports.test.ts @@ -0,0 +1,81 @@ +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { it } from "#veryfront/testing/bdd.ts"; + +type Module = { + specifier: string; + error?: unknown; + dependencies?: { code?: { specifier: string } }[]; +}; +type Graph = { roots: string[]; modules: Module[] }; +const root = fileURLToPath(new URL("../../../", import.meta.url)); +const forbidden = [ + "/src/config/loader.ts", + "/src/agent/factory.ts", + "/src/tool/factory.ts", + "/src/agent/project/agent-runtime.ts", + "/src/agent/service/routes.ts", + "/src/agent/hosted/cloud-agent-config.ts", + "/src/agent/hosted/default-chat-runtime.ts", + "/src/agent/hosted/executor-discovery-node.ts", + "/src/server/runtime-handler/index.ts", + "/src/agent/hosted/executor-runtime-entrypoint.ts", + "/src/agent/hosted/executor-runtime-facades.ts", +]; + +for ( + const entry of [ + "src/agent/hosted/managed-executor-broker.ts", + "src/agent/service/broker-ingress.ts", + "src/agent/service/managed-broker-handler.ts", + "src/agent/service/managed-broker.ts", + "src/agent/service/managed-node-broker.ts", + ] +) { + it(`keeps project runtime imports out of ${entry}`, { timeout: 30_000 }, async () => { + const child = spawn(typeof Deno === "undefined" ? "deno" : Deno.execPath(), [ + "info", + "--frozen", + "--json", + entry, + ], { cwd: root, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + let diagnostics = ""; + child.stdout.on("data", (chunk) => output += chunk); + child.stderr.on("data", (chunk) => diagnostics += chunk); + const timer = setTimeout(() => child.kill(), 25_000); + try { + const code = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", resolve); + }); + assertEquals(code, 0, diagnostics); + const graph: Graph = JSON.parse(output); + const modules = new Map(graph.modules.map((module) => [module.specifier, module])); + const visited = new Set(); + const pending = [...graph.roots]; + while (pending.length) { + const specifier = pending.shift()!; + if (visited.has(specifier)) continue; + visited.add(specifier); + // Include statically resolvable dynamic imports; omit erased type edges. + for (const dependency of modules.get(specifier)?.dependencies ?? []) { + if (dependency.code) pending.push(dependency.code.specifier); + } + } + assertEquals( + [...visited].filter((specifier) => modules.get(specifier)?.error !== undefined), + [], + "Runtime imports must resolve before the boundary graph can pass", + ); + assertEquals( + [...visited].filter((specifier) => forbidden.some((path) => specifier.endsWith(path))), + [], + ); + } finally { + clearTimeout(timer); + child.kill(); + } + }); +} diff --git a/tests/integration/agent/managed-broker-persistence.test.ts b/tests/integration/agent/managed-broker-persistence.test.ts new file mode 100644 index 0000000000..d35a5156fd --- /dev/null +++ b/tests/integration/agent/managed-broker-persistence.test.ts @@ -0,0 +1,372 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects, assertThrows } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { withMockFetch } from "#veryfront/testing/mock-fetch.ts"; +import { createManagedBrokerPersistence } from "#veryfront/agent/hosted/managed-broker-persistence.ts"; +import { FakeTime } from "#std/testing/time"; + +const conversationId = "00000000-0000-4000-8000-000000000001"; +const messageId = "00000000-0000-4000-8000-000000000002"; +const run = { + runId: "run-1", + conversationId, + messageId, + latestEventId: 0, + latestExternalEventSequence: 0, + waitingToolCallId: null, + waitingToolName: null, + status: "running" as const, + streamProtocolVersion: 2 as const, +}; + +function successfulFetch(calls: Record[]) { + let cursor = 0; + return async (_input: RequestInfo | URL, init?: RequestInit) => { + const body = init?.body ? JSON.parse(String(init.body)) as Record : {}; + calls.push(body); + if (Array.isArray(body.events)) { + cursor += body.events.length; + return Response.json({ + latest_event_id: cursor, + latest_external_event_sequence: cursor, + appended_count: body.events.length, + run: { + run_id: run.runId, + conversation_id: conversationId, + latest_event_id: cursor, + latest_external_event_sequence: cursor, + }, + }); + } + return Response.json({ completed: true, run: { runId: run.runId, status: body.status } }); + }; +} + +function bindForTest( + persistence: ReturnType, +): ReturnType { + persistence.bindSessionOwnedWork(async (operation) => await operation()); + return persistence; +} + +describe("managed broker persistence", () => { + it("requires one active session owner before persistence can enqueue or fetch", async () => { + const calls: Record[] = []; + const persistence = createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch: successfulFetch(calls), + }); + + await assertRejects(async () => + await persistence.modelRunEventSink({ + type: "AGENT_RUN_MODEL_CALL_CONTEXT", + messages: [], + tools: [], + }) + ); + await assertRejects(() => persistence.publishParentRunEvents([{ type: "STEP_STARTED" }])); + await assertRejects(() => + persistence.persistToolExposureCheckpoint({ + version: 2, + loadedToolNames: ["search"], + }) + ); + await assertRejects(() => + persistence.persistProviderReplayCheckpoint({ + version: 1, + messageId, + provider: "anthropic", + providerBlocks: [], + providerBlockPositions: [], + providerMessageBlockCounts: [], + totalPartCount: 0, + }) + ); + await assertRejects(() => + persistence.output.write({ type: "text-delta", id: "message", delta: "blocked" }) + ); + await assertRejects(() => persistence.output.finish({ completed: false })); + assertEquals(calls, []); + + const owner = async (operation: () => Promise): Promise => await operation(); + assertThrows(() => persistence.bindSessionOwnedWork(undefined as never)); + persistence.bindSessionOwnedWork(owner); + assertThrows(() => persistence.bindSessionOwnedWork(owner)); + await persistence.cleanup(); + + const cleaned = createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch: successfulFetch([]), + }); + await cleaned.cleanup(); + assertThrows(() => cleaned.bindSessionOwnedWork(owner)); + }); + + it("persists output, audit, parent events, checkpoints, and terminal completion", async () => { + const calls: Record[] = []; + const fetch = successfulFetch(calls); + await withMockFetch( + () => Promise.reject(new Error("external fetch must not be used")), + async () => { + const persistence = bindForTest(createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "veryfront-cloud/openai/synthetic", + resolveProvider: () => "openai", + fetch, + })); + await persistence.output.write({ type: "text-delta", id: "message", delta: "hello" }); + await persistence.modelRunEventSink({ + type: "AGENT_RUN_MODEL_CALL_CONTEXT", + messages: [], + tools: [], + }); + await persistence.publishParentRunEvents([{ type: "STEP_STARTED" }]); + await persistence.persistToolExposureCheckpoint({ + version: 2, + loadedToolNames: ["search"], + }); + await persistence.persistProviderReplayCheckpoint({ + version: 1, + messageId, + provider: "anthropic", + providerBlocks: [{ + type: "provider-block", + provider: "anthropic", + block: { type: "redacted_thinking", data: "synthetic" }, + }], + providerBlockPositions: [0], + providerMessageBlockCounts: [1], + totalPartCount: 1, + }); + await persistence.output.finish({ + completed: true, + metadata: { + modelId: "veryfront-cloud/openai/synthetic", + usage: { inputTokens: 12, outputTokens: 7, cachedInputTokens: 3 }, + usageCaptureStatus: "complete", + }, + }); + await assertRejects(() => persistence.publishParentRunEvents([{ type: "STEP_FINISHED" }])); + await persistence.cleanup(); + }, + ); + const events = calls.flatMap((call) => Array.isArray(call.events) ? call.events : []); + assertEquals(events.some((event) => event.type === "TEXT_MESSAGE_CONTENT"), true); + assertEquals(events.some((event) => event.type === "AGENT_RUN_MODEL_CALL_CONTEXT"), true); + assertEquals(events.some((event) => event.type === "STEP_STARTED"), true); + assertEquals(events.some((event) => event.type === "AGENT_RUN_TOOL_EXPOSURE_CHECKPOINT"), true); + assertEquals( + events.some((event) => event.type === "AGENT_RUN_PROVIDER_REPLAY_CHECKPOINT"), + true, + ); + assertEquals(calls.at(-1)?.status, "completed"); + assertEquals(calls.at(-1)?.metadata, { + provider: "openai", + model: "veryfront-cloud/openai/synthetic", + inputTokens: 12, + outputTokens: 7, + usageCaptureStatus: "complete", + finishReason: "stop", + }); + }); + + it("retains a queued cancellation finish until the original output write settles", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const calls: Record[] = []; + const fallback = successfulFetch(calls); + let first = true; + const fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + if (first) { + first = false; + entered.resolve(); + return await release.promise; + } + return await fallback(input, init); + }; + await withMockFetch(fetch, async () => { + const persistence = bindForTest(createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch, + })); + const write = persistence.output.write({ + type: "text-delta", + id: "message", + delta: "pending", + }); + await entered.promise; + let finished = false; + const finish = persistence.output.finish({ completed: false }).then(() => finished = true); + await Promise.resolve(); + assertEquals(finished, false); + release.resolve(Response.json({ + latest_event_id: 1, + latest_external_event_sequence: 1, + appended_count: 1, + run: { + run_id: run.runId, + conversation_id: conversationId, + latest_event_id: 1, + latest_external_event_sequence: 1, + }, + })); + await write; + await finish; + assertEquals(calls.at(-1)?.status, "cancelled"); + await persistence.cleanup(); + }); + }); + + it("preserves a poisoned write error while independently finalizing the run as failed", async () => { + const calls: Record[] = []; + const fallback = successfulFetch(calls); + let failEventAppend = true; + const fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const body = init?.body ? JSON.parse(String(init.body)) as Record : {}; + if (failEventAppend && Array.isArray(body.events)) { + failEventAppend = false; + calls.push(body); + return new Response("failed", { status: 500 }); + } + return await fallback(input, init); + }; + await withMockFetch(fetch, async () => { + const persistence = bindForTest(createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch, + })); + const write = persistence.output.write({ + type: "text-delta", + id: "message", + delta: "fail", + }); + const finish = persistence.output.finish({ completed: true }); + const [writeResult, finishResult] = await Promise.allSettled([write, finish]); + const writeError = writeResult.status === "rejected" ? writeResult.reason : undefined; + assertEquals(writeError instanceof Error, true); + const finishError = finishResult.status === "rejected" ? finishResult.reason : undefined; + assertEquals(finishError === writeError, true); + assertEquals(calls.at(-1)?.status, "failed"); + await persistence.cleanup(); + }); + }); + + it("retains a noncooperative model audit append after its deadline failure", async () => { + using time = new FakeTime(); + const appendEntered = Promise.withResolvers(); + const appendRelease = Promise.withResolvers(); + const calls: Record[] = []; + const fallback = successfulFetch(calls); + let delayAuditAppend = true; + const fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const body = init?.body ? JSON.parse(String(init.body)) as Record : {}; + if (delayAuditAppend && Array.isArray(body.events)) { + delayAuditAppend = false; + appendEntered.resolve(); + return await appendRelease.promise; + } + return await fallback(input, init); + }; + const persistence = createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch, + }); + let ownedWorkTail = Promise.resolve(); + persistence.bindSessionOwnedWork((operation) => { + const result = operation(); + const settled = result.then(() => undefined, () => undefined); + ownedWorkTail = Promise.all([ownedWorkTail, settled]).then(() => undefined); + return result; + }); + const audit = persistence.modelRunEventSink({ + type: "AGENT_RUN_MODEL_CALL_CONTEXT", + messages: [], + tools: [], + }); + await appendEntered.promise; + + time.tick(30_000); + const auditResult = await Promise.allSettled([audit]); + const auditError = auditResult[0]?.status === "rejected" ? auditResult[0].reason : undefined; + assertEquals(auditError instanceof Error, true); + assertEquals((auditError as Error).message, "Durable run event persistence timed out"); + + const finishResult = await Promise.allSettled([ + persistence.output.finish({ completed: false, error: auditError }), + ]); + assertEquals(finishResult[0]?.status, "rejected"); + assertEquals( + finishResult[0]?.status === "rejected" ? finishResult[0].reason : undefined, + auditError, + ); + assertEquals(calls.at(-1)?.status, "failed"); + + let ownedWorkSettled = false; + void ownedWorkTail.then(() => ownedWorkSettled = true); + let cleanupSettled = false; + const cleanup = persistence.cleanup().then(() => cleanupSettled = true); + for (let index = 0; index < 20; index += 1) await Promise.resolve(); + assertEquals(ownedWorkSettled, false); + assertEquals(cleanupSettled, false); + + appendRelease.resolve(Response.json({ + latest_event_id: 1, + latest_external_event_sequence: 1, + appended_count: 1, + run: { + run_id: run.runId, + conversation_id: conversationId, + latest_event_id: 1, + latest_external_event_sequence: 1, + }, + })); + await ownedWorkTail; + assertEquals(ownedWorkSettled, true); + await cleanup; + assertEquals(cleanupSettled, true); + }); + + it("persists a failed terminal outcome for executor output errors", async () => { + const calls: Record[] = []; + const fetch = successfulFetch(calls); + await withMockFetch(fetch, async () => { + const persistence = bindForTest(createManagedBrokerPersistence({ + apiUrl: "https://api.example.test", + runEventToken: "run-event-token", + run, + modelId: "model", + resolveProvider: () => "provider", + fetch, + })); + await persistence.output.finish({ + completed: false, + error: new Error("synthetic execution failure"), + }); + assertEquals(calls.at(-1)?.status, "failed"); + assertEquals(calls.at(-1)?.terminal_error_code, "STREAM_ERROR"); + await persistence.cleanup(); + }); + }); +}); diff --git a/tests/integration/agent/managed-broker-project-state.test.ts b/tests/integration/agent/managed-broker-project-state.test.ts new file mode 100644 index 0000000000..c658f17438 --- /dev/null +++ b/tests/integration/agent/managed-broker-project-state.test.ts @@ -0,0 +1,223 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects, assertStrictEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { createManagedBrokerProjectState } from "veryfront/agent/managed-broker"; + +const definition = { + id: "coder", + name: "Coder", + description: "Codes", + instructions: "Base instructions", + skills: false as const, +}; + +describe("managed broker project state", () => { + for (const availableToolNames of [undefined, [], ["read_file"], ["load_skill"]]) { + it(`keeps refreshed skills gated by the effective tool selection: ${availableToolNames}`, async () => { + const marker = "Synthetic selected skill catalog marker"; + const state = createManagedBrokerProjectState({ + apiUrl: "https://api.example.test", + authToken: "broker-token", + agentId: "coder", + projectId: "project-1", + builtinSkills: [{ + id: "guide", + name: "Guide", + description: marker, + instructions: "Help.", + allowedTools: [], + }], + fetch: (value) => + Promise.resolve( + new URL(value).pathname.endsWith("/AGENTS.md") + ? Response.json({ path: "AGENTS.md", content: "Current project instructions" }) + : Response.json({ data: [], page_info: { next: null } }), + ), + }); + const prepared = await state.prepareProjectSteering({ + definition: { ...definition, skills: true }, + projectId: "project-1", + signal: new AbortController().signal, + }); + assertEquals(prepared.initialSkills?.length, 1); + const refreshed = await state.refreshProjectSteering( + new AbortController().signal, + availableToolNames, + ); + assertEquals(JSON.stringify(refreshed).includes("Current project instructions"), true); + assertEquals( + JSON.stringify(refreshed).includes(marker), + availableToolNames?.includes("load_skill") ?? false, + ); + }); + } + + it("joins the original catalog lookup before propagating an instruction failure", async () => { + const failure = new Error("synthetic instruction failure"); + const catalog = Promise.withResolvers(); + let listingCalls = 0; + const state = createManagedBrokerProjectState({ + apiUrl: "https://api.example.test", + authToken: "broker-token", + agentId: "coder", + projectId: "project-1", + fetch: (value) => { + const url = new URL(value); + if (url.pathname.endsWith("/AGENTS.md")) return Promise.reject(failure); + listingCalls++; + return listingCalls === 1 + ? catalog.promise + : Promise.resolve(Response.json({ data: [], page_info: { next: null } })); + }, + }); + let rejected = false; + const pending = state.prepareProjectSteering({ + definition, + projectId: "project-1", + signal: new AbortController().signal, + }).catch((error) => { + rejected = true; + throw error; + }); + void pending.catch(() => {}); + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(rejected, false); + catalog.resolve(Response.json({ data: [], page_info: { next: null } })); + assertStrictEquals(await assertRejects(() => pending), failure); + }); + + it("uses fixed project authorization and performs a complete refresh", async () => { + const calls: Array<{ url: URL; authorization: string | null }> = []; + let instructionRead = 0; + const fetch = (value: string, init: RequestInit) => { + const url = new URL(value); + calls.push({ url, authorization: new Headers(init.headers).get("authorization") }); + if (url.pathname.endsWith("/AGENTS.md")) { + instructionRead++; + return Promise.resolve(Response.json({ + path: "AGENTS.md", + content: `Project instructions ${instructionRead}`, + })); + } + return Promise.resolve(Response.json({ data: [], page_info: { next: null } })); + }; + const state = createManagedBrokerProjectState({ + apiUrl: "https://api.example.test", + authToken: "broker-token", + agentId: "coder", + projectId: "project-1", + branchId: "branch-1", + fetch, + }); + const prepared = await state.prepareProjectSteering({ + definition, + projectId: "project-1", + branchId: "branch-1", + signal: new AbortController().signal, + }); + assertEquals(prepared.initialProjectInstructions, "Project instructions 1"); + const refreshed = await state.refreshProjectSteering(new AbortController().signal); + assertEquals(JSON.stringify(refreshed).includes("Project instructions 2"), true); + assertEquals(calls.every((call) => call.authorization === "Bearer broker-token"), true); + assertEquals(calls.every((call) => call.url.origin === "https://api.example.test"), true); + await assertRejects(() => + state.prepareProjectSteering({ + definition, + projectId: "project-2", + branchId: "branch-1", + signal: new AbortController().signal, + }) + ); + }); + + it("uses no API lookup for a null project and forwards conversation cancellation", async () => { + let fetches = 0; + let conversationSignal: AbortSignal | undefined; + const state = createManagedBrokerProjectState({ + apiUrl: "https://api.example.test", + authToken: "broker-token", + agentId: "coder", + projectId: null, + fetch: () => { + fetches++; + return Promise.reject(new Error("must not fetch")); + }, + latestConversationUserText: (signal) => { + conversationSignal = signal; + return Promise.resolve(null); + }, + }); + const prepared = await state.prepareProjectSteering({ + definition, + projectId: null, + signal: new AbortController().signal, + }); + assertEquals(prepared.agent.id, "coder"); + await state.refreshProjectSteering(new AbortController().signal); + const controller = new AbortController(); + controller.abort(); + await assertRejects(() => state.latestConversationUserText!(controller.signal)); + assertEquals(conversationSignal, undefined); + assertEquals(fetches, 0); + }); + + it("cancels in-flight project helper reads", async () => { + const entered = Promise.withResolvers(); + const state = createManagedBrokerProjectState({ + apiUrl: "https://api.example.test", + authToken: "broker-token", + agentId: "coder", + projectId: "project-1", + fetch: (_url, init) => { + const signal = init.signal!; + entered.resolve(signal); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + }, + }); + const controller = new AbortController(); + const pending = state.prepareProjectSteering({ + definition, + projectId: "project-1", + signal: controller.signal, + }); + const signal = await entered.promise; + controller.abort(); + await assertRejects(() => pending); + assertEquals(signal.aborted, true); + }); + + it("keeps local discovery and factories outside its dependency graph", async () => { + const output = await new Deno.Command(Deno.execPath(), { + cwd: new URL("../../../", import.meta.url), + args: ["info", "--frozen", "--json", "src/agent/hosted/managed-broker-project-state.ts"], + }).output(); + assertEquals(output.code, 0, new TextDecoder().decode(output.stderr)); + const graph = JSON.parse(new TextDecoder().decode(output.stdout)) as { + roots: string[]; + modules: Array<{ specifier: string; dependencies?: Array<{ code?: { specifier: string } }> }>; + }; + const modules = new Map(graph.modules.map((module) => [module.specifier, module])); + const visited = new Set(); + const pending = [...graph.roots]; + while (pending.length) { + const specifier = pending.shift()!; + if (visited.has(specifier)) continue; + visited.add(specifier); + for (const dependency of modules.get(specifier)?.dependencies ?? []) { + if (dependency.code) pending.push(dependency.code.specifier); + } + } + const forbidden = [ + "/src/config/loader.ts", + "/src/agent/factory.ts", + "/src/tool/factory.ts", + "/src/agent/hosted/executor-discovery-node.ts", + ]; + assertEquals( + [...visited].filter((specifier) => forbidden.some((path) => specifier.endsWith(path))), + [], + ); + }); +}); diff --git a/tests/integration/agent/managed-node-broker.test.ts b/tests/integration/agent/managed-node-broker.test.ts new file mode 100644 index 0000000000..85e9ca607f --- /dev/null +++ b/tests/integration/agent/managed-node-broker.test.ts @@ -0,0 +1,292 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { startNodeManagedAgentBroker } from "#veryfront/agent/service/managed-node-broker.ts"; + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +describe("managed node broker", () => { + it("routes every required managed surface and preserves health contracts", async () => { + const calls: Array<{ name: string; runId?: string; body: string }> = []; + const handler = (name: string) => ({ + async handle(request: Request, input: { runId?: string }) { + calls.push({ + name, + ...(input.runId ? { runId: input.runId } : {}), + body: await request.text(), + }); + return Response.json({ name, runId: input.runId }); + }, + }); + const server = await startNodeManagedAgentBroker({ + port: 0, + bindAddress: "127.0.0.1", + signals: [], + readiness: () => true, + broker: { + shutdown: () => Promise.resolve(), + closed: Promise.resolve(), + settled: Promise.resolve(), + }, + handlers: { + signedStream: handler("signed"), + durableStart: handler("start"), + agUi: handler("ag-ui"), + cancel: handler("cancel"), + resume: handler("resume"), + }, + }); + try { + assertEquals(await (await fetch(`${server.url}/liveness`)).text(), "OK"); + assertEquals(await (await fetch(`${server.url}/readiness`)).text(), "OK"); + for ( + const [method, path, body] of [ + ["POST", "/api/control-plane/runs/run-1/stream", "signed-body"], + ["POST", "/api/runs", "start-body"], + ["POST", "/api/ag-ui", "ag-ui-body"], + ["DELETE", "/api/runs/run-2", ""], + ["POST", "/api/runs/run-3/resume", "resume-direct"], + ["DELETE", "/api/control-plane/runs/run-4", ""], + ["POST", "/api/control-plane/runs/run-5/resume", "resume-control"], + ] + ) { + const response = await fetch(`${server.url}${path}`, { method, body: body || undefined }); + assertEquals(response.status, 200); + await response.body?.cancel(); + } + assertEquals(calls, [ + { name: "signed", runId: "run-1", body: "signed-body" }, + { name: "start", body: "start-body" }, + { name: "ag-ui", body: "ag-ui-body" }, + { name: "cancel", runId: "run-2", body: "" }, + { name: "resume", runId: "run-3", body: "resume-direct" }, + { name: "cancel", runId: "run-4", body: "" }, + { name: "resume", runId: "run-5", body: "resume-control" }, + ]); + } finally { + await server.stop(); + } + }); + + it("rejects malformed run IDs before dispatching managed handlers", async () => { + const runIds: string[] = []; + const handler = { + handle(_request: Request, input: { runId?: string }) { + runIds.push(input.runId!); + return new Response("handled"); + }, + }; + const server = await startNodeManagedAgentBroker({ + port: 0, + bindAddress: "127.0.0.1", + signals: [], + readiness: () => true, + broker: { + shutdown: () => Promise.resolve(), + closed: Promise.resolve(), + settled: Promise.resolve(), + }, + handlers: { + signedStream: handler, + durableStart: handler, + agUi: handler, + cancel: handler, + resume: handler, + }, + }); + try { + const routes = [ + ["POST", "/api/control-plane/runs/", "/stream"], + ["POST", "/api/runs/", "/resume"], + ["POST", "/api/control-plane/runs/", "/resume"], + ["DELETE", "/api/runs/", ""], + ["DELETE", "/api/control-plane/runs/", ""], + ] as const; + for (const [method, prefix, suffix] of routes) { + for ( + const malformed of [ + "%", + "%GG", + "%E0%A4%A", + "%2F", + "%00", + "%5C", + "%20", + "%C3%A9", + "%252F", + "a".repeat(129), + ] + ) { + const response = await fetch(`${server.url}${prefix}${malformed}${suffix}`, { method }); + const body = await response.text(); + assertEquals(response.status, 400); + assertEquals(JSON.parse(body), { errorCode: "BROKER_INGRESS_TARGET_MISMATCH" }); + } + } + assertEquals(runIds, []); + for (const [method, prefix, suffix] of routes) { + const response = await fetch(`${server.url}${prefix}%72un%2D1${suffix}`, { method }); + assertEquals(await response.text(), "handled"); + assertEquals(response.status, 200); + } + assertEquals(runIds, ["run-1", "run-1", "run-1", "run-1", "run-1"]); + } finally { + await server.stop(); + } + }); + + it("stops admission before joining handler and broker retirement", async () => { + const events: string[] = []; + const retirement = Promise.withResolvers(); + const shared = { + handle: () => Promise.resolve(new Response("handled")), + close: () => { + events.push("handler-close"); + }, + }; + const server = await startNodeManagedAgentBroker({ + port: 0, + bindAddress: "127.0.0.1", + signals: [], + readiness: () => true, + broker: { + shutdown() { + events.push("broker-shutdown"); + return Promise.resolve(); + }, + closed: Promise.resolve(), + settled: retirement.promise, + }, + handlers: { + signedStream: shared, + durableStart: shared, + agUi: shared, + cancel: shared, + resume: shared, + }, + }); + let stopped = false; + const stopping = server.stop().then(() => stopped = true); + await tick(); + assertEquals(events[0], "broker-shutdown"); + assertEquals(stopped, false); + retirement.resolve(); + await stopping; + assertEquals(stopped, true); + assertEquals(events, ["broker-shutdown", "handler-close"]); + }); + + for (const shutdownThrows of [false, true]) { + it(`joins retirement after synchronous close failure (shutdown throws: ${shutdownThrows})`, async () => { + const failure = new Error("synthetic close failure"); + const retirement = Promise.withResolvers(); + const firstClosed = Promise.withResolvers(); + let secondClosed = false; + const first = { + handle: () => new Response("handled"), + close: () => { + firstClosed.resolve(); + throw failure; + }, + }; + const second = { + handle: () => new Response("handled"), + close: () => { + secondClosed = true; + return retirement.promise; + }, + }; + const server = await startNodeManagedAgentBroker({ + port: 0, + bindAddress: "127.0.0.1", + signals: [], + readiness: () => true, + broker: { + shutdown: () => { + if (shutdownThrows) throw failure; + return Promise.resolve(); + }, + closed: Promise.resolve(), + settled: retirement.promise, + }, + handlers: { + signedStream: first, + durableStart: second, + agUi: second, + cancel: second, + resume: second, + }, + }); + let stopped = false; + const stopping = server.stop().then( + () => { + stopped = true; + return undefined; + }, + (error: unknown) => { + stopped = true; + return error; + }, + ); + try { + await Promise.race([firstClosed.promise, stopping]); + assertEquals(secondClosed, true); + assertEquals(stopped, false); + retirement.resolve(); + assertEquals(await stopping, failure); + } finally { + retirement.resolve(); + await stopping; + } + }); + } + + it("rejects incomplete route configuration before binding", async () => { + await assertRejects(() => + startNodeManagedAgentBroker({ + port: 0, + signals: [], + readiness: () => true, + broker: { + shutdown: () => Promise.resolve(), + closed: Promise.resolve(), + settled: Promise.resolve(), + }, + handlers: {} as never, + }) + ); + }); + + it("keeps project loaders and agent factories outside its dependency graph", async () => { + const output = await new Deno.Command(Deno.execPath(), { + cwd: new URL("../../../", import.meta.url), + args: ["info", "--frozen", "--json", "src/agent/service/managed-node-broker.ts"], + }).output(); + assertEquals(output.code, 0, new TextDecoder().decode(output.stderr)); + const graph = JSON.parse(new TextDecoder().decode(output.stdout)) as { + roots: string[]; + modules: Array<{ specifier: string; dependencies?: Array<{ code?: { specifier: string } }> }>; + }; + const modules = new Map(graph.modules.map((module) => [module.specifier, module])); + const visited = new Set(); + const pending = [...graph.roots]; + while (pending.length) { + const specifier = pending.shift()!; + if (visited.has(specifier)) continue; + visited.add(specifier); + for (const dependency of modules.get(specifier)?.dependencies ?? []) { + if (dependency.code) pending.push(dependency.code.specifier); + } + } + const forbidden = [ + "/src/config/loader.ts", + "/src/agent/factory.ts", + "/src/tool/factory.ts", + "/src/agent/project/agent-runtime.ts", + ]; + assertEquals( + [...visited].filter((specifier) => forbidden.some((path) => specifier.endsWith(path))), + [], + ); + }); +}); diff --git a/tests/integration/agent/remote-source-projection-intrinsics.test.ts b/tests/integration/agent/remote-source-projection-intrinsics.test.ts new file mode 100644 index 0000000000..baff363234 --- /dev/null +++ b/tests/integration/agent/remote-source-projection-intrinsics.test.ts @@ -0,0 +1,72 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { + bindRuntimeRemoteToolSourcesToCredentialOwner, + constrainRuntimeRemoteToolSources, + getRuntimeRemoteToolSources, + type RuntimeRemoteToolConfig, + VERYFRONT_API_MCP_SOURCE_ID, +} from "#veryfront/agent/runtime/mcp-server-tool-sources.ts"; +import type { AgentConfig } from "#veryfront/agent/types.ts"; +import type { RemoteToolSource } from "#veryfront/tool"; +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; + +for (const explicitPolicy of [false, true]) { + for (const hook of [undefined, "map", "filter", "some", Symbol.iterator] as const) { + describe(`private injected remote sources ${explicitPolicy ? "explicit" : "runtime"} ${String(hook ?? "baseline")}`, () => { + it("keeps raw execution capabilities private while preserving the selected policy", async () => { + const calls: string[] = []; + const source: RemoteToolSource = { + id: VERYFRONT_API_MCP_SOURCE_ID, + listTools: async () => [], + executeTool: async (name) => { + calls.push(name); + return { success: true, data: "synthetic-result" }; + }, + }; + const config: AgentConfig & RuntimeRemoteToolConfig = { + id: "private-remote-agent", + system: "Synthetic remote tool test", + tools: true, + ...(explicitPolicy + ? { mcpServers: [{ kind: "veryfront-api", toolPolicy: { allow: ["allowed"] } }] } + : {}), + __vfRemoteToolSources: [source], + }; + const apply = Reflect.apply; + const defineProperty = Object.defineProperty; + const descriptor = hook === undefined + ? undefined + : Object.getOwnPropertyDescriptor(Array.prototype, hook)!; + let leaked: RemoteToolSource | undefined; + let selected: RemoteToolSource[] | undefined; + try { + if (hook !== undefined) { + defineProperty(Array.prototype, hook, { + ...descriptor, + value: function (this: unknown[], ...args: unknown[]) { + for (let i = 0; i < this.length; i++) { + if (this[i] === source) leaked = source; + } + return apply(descriptor!.value, this, args); + }, + }); + } + selected = getRuntimeRemoteToolSources(config); + selected = bindRuntimeRemoteToolSourcesToCredentialOwner(selected, { + agentId: "private-remote-agent", + }); + selected = constrainRuntimeRemoteToolSources(selected, ["allowed"]); + } finally { + if (hook !== undefined) defineProperty(Array.prototype, hook, descriptor!); + } + assertEquals(selected?.length, 1); + await selected![0]!.executeTool("allowed", {}); + await assertRejects(async () => await selected![0]!.executeTool("denied", {})); + if (leaked) await leaked.executeTool("denied", {}); + assertEquals(calls, ["allowed"]); + assertEquals(leaked, undefined); + }); + }); + } +} diff --git a/tests/integration/agent/runtime-private-reflection.test.ts b/tests/integration/agent/runtime-private-reflection.test.ts new file mode 100644 index 0000000000..7e106b2cdb --- /dev/null +++ b/tests/integration/agent/runtime-private-reflection.test.ts @@ -0,0 +1,101 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { createEphemeralAgentWithRuntimeOptions } from "#veryfront/agent/factory.ts"; +import type { RuntimeRemoteToolConfig } from "#veryfront/agent/runtime/mcp-server-tool-sources.ts"; +import type { AgentConfig } from "#veryfront/agent/types.ts"; +import { scriptedModel } from "#veryfront/agent/runtime/model-runtime.test-helpers.ts"; +import { assertEquals, assertStringIncludes } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; + +for (const mode of ["generate", "stream"] as const) { + for (const hooks of [false, true]) { + describe(`private runtime reflection ${mode} ${hooks ? "hooks" : "baseline"}`, () => { + for (const subject of ["prompt", "arguments", "form"] as const) { + it(`preserves ${subject} execution without exposing private values`, async () => { + const marker = "synthetic-private-runtime-reflection"; + const name = subject === "form" ? "form_input" : "inspect"; + const model = scriptedModel( + subject === "prompt" ? [{ text: "Complete" }] : [ + { + toolCalls: [{ + id: "call", + name, + input: JSON.stringify({ text: subject === "arguments" ? marker : "public" }), + }], + }, + { text: "Complete" }, + ], + { only: mode }, + ); + let executions = 0; + const config: AgentConfig & RuntimeRemoteToolConfig = { + model: "veryfront-cloud/openai/gpt-5.4", + system: subject === "prompt" ? marker : "Synthetic instructions", + skills: false, + maxSteps: 3, + tools: subject === "prompt" ? {} : { [name]: true }, + __vfRemoteToolSources: subject === "prompt" ? [] : [{ + id: "synthetic-remote-source", + listTools: async () => [{ + name, + description: "Synthetic tool", + parameters: { + type: "object", + properties: { text: { type: "string" } }, + required: ["text"], + }, + }], + executeTool: async () => { + executions++; + return subject === "form" + ? JSON.stringify({ nested: { submitted: true, answer: marker } }) + : { ok: true }; + }, + }], + }; + const runtime = createEphemeralAgentWithRuntimeOptions(config, { + resolveModelRuntime: () => model, + }); + const apply = Reflect.apply; + const stringify = JSON.stringify; + const includes = String.prototype.includes; + const defineProperty = Object.defineProperty; + const methods = [ + { target: Array, key: "isArray", argument: true }, + { target: Object, key: "values", argument: true }, + { target: Array.prototype, key: "some", argument: false }, + ].map((method) => ({ + ...method, + descriptor: Object.getOwnPropertyDescriptor(method.target, method.key)!, + })); + let observations = 0; + let output = ""; + try { + if (hooks) { + for (const method of methods) { + defineProperty(method.target, method.key, { + ...method.descriptor, + value: function (this: unknown, ...args: unknown[]) { + if ( + apply(includes, stringify(method.argument ? args[0] : this) ?? "", [marker]) + ) observations++; + return apply(method.descriptor.value, this, args); + }, + }); + } + } + output = mode === "stream" + ? await (await runtime.stream({ input: "Run" })).toDataStreamResponse().text() + : (await runtime.generate({ input: "Run" })).text; + } finally { + for (const method of methods) { + defineProperty(method.target, method.key, method.descriptor); + } + } + assertEquals(executions, subject === "prompt" ? 0 : 1); + assertStringIncludes(output, "Complete"); + assertEquals(observations, 0); + }); + } + }); + } +} diff --git a/tests/integration/ci/node-executor-coverage.test.ts b/tests/integration/ci/node-executor-coverage.test.ts index 50cbc0c806..6e52504412 100644 --- a/tests/integration/ci/node-executor-coverage.test.ts +++ b/tests/integration/ci/node-executor-coverage.test.ts @@ -11,6 +11,70 @@ import { const root = fileURLToPath(new URL("../../../", import.meta.url)); describe("native executor source coverage", () => { + it("retains an original function anchor when generated coordinates repeat its name", async () => { + await Deno.mkdir(`${root}/coverage`, { recursive: true }); + const directory = await makeTempDirWithOptions({ + dir: `${root}/coverage`, + prefix: "node-body-anchor-", + }); + const source = `${directory}/body-anchor.ts`; + const lcov = `${directory}/lcov.info`; + await Deno.writeTextFile( + source, + [ + "/** Typed declaration whose first executable range starts later. */", + "export function bodyMappedTarget(value: number): number {", + " if (value > 0) {", + " return value;", + " }", + " return 0;", + "}", + "", + "export const trailingTopLevel = 1;", + ].join("\n"), + ); + await Deno.writeTextFile( + lcov, + [ + "TN:", + `SF:${source}`, + "FN:4,bodyMappedTarget", + "FN:1,bodyMappedTarget", + "FNDA:1,bodyMappedTarget", + "DA:4,1", + "end_of_record", + "", + ].join("\n"), + ); + try { + const summaries = await validateNativeCoverage({ + root, + reportPath: lcov, + sourceFiles: [source], + }); + assertEquals(summaries, [{ source, linesHit: 1, linesFound: 1 }]); + await Deno.writeTextFile( + lcov, + [ + "TN:", + `SF:${source}`, + "FN:9,bodyMappedTarget", + "FNDA:1,bodyMappedTarget", + "DA:9,1", + "end_of_record", + "", + ].join("\n"), + ); + await assertRejects( + () => validateNativeCoverage({ root, reportPath: lcov, sourceFiles: [source] }), + Error, + "not mapped to original source", + ); + } finally { + await Deno.remove(directory, { recursive: true }); + } + }); + it("maps real TypeScript coverage and rejects transformed JavaScript positions", async () => { await Deno.mkdir(`${root}/coverage`, { recursive: true }); const directory = await makeTempDirWithOptions({ @@ -72,7 +136,7 @@ describe("native executor source coverage", () => { sourceFiles: [source], }); assertEquals(summaries.length, 1); - assert(summaries[0].linesHit > 0); + assert(summaries[0]!.linesHit > 0); } else { await assertRejects( () => validateNativeCoverage({ root, reportPath: lcov, sourceFiles: [source] }),