From f1ff7087cc683cdb2c8128eb06b644929da25ca1 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 06:26:01 +0000 Subject: [PATCH 01/13] fix(eval): restore the headless coding tool profile Freeze a versioned HostedExecution tool ceiling for Maka benchmark subjects while keeping Runtime Host as the sole execution authority.\n\nGenerated-by: Codex --- packages/eval/README.md | 7 +++ ...bench-2.1-deepseek-v4-flash-eight-arm.json | 3 +- ...-bench-2.1-deepseek-v4-flash-four-arm.json | 3 +- .../__tests__/lifecycle-boundaries.test.ts | 9 +++- packages/eval/src/maka-subject.ts | 13 +++++- .../execution-model-composition.test.ts | 27 ++++++++++++ .../hosted-execution-tool-profile.test.ts | 42 ++++++++++++++++++ .../src/protocol/hosted-execution.ts | 19 +++++++- .../src/server/execution-composition.ts | 9 +++- .../server/hosted-execution-tool-profile.ts | 43 +++++++++++++++++++ .../src/server/interactive-run-composer.ts | 39 +++++++++++++++-- 11 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts create mode 100644 packages/runtime-host/src/server/hosted-execution-tool-profile.ts diff --git a/packages/eval/README.md b/packages/eval/README.md index 3632d9ed90..7ebbed3ed3 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -26,4 +26,11 @@ The result kernel contains only score, normalized usage, attributable cost, dura The checked-in Terminal-Bench 2.1 four-arm cohort is `experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json`. It freezes provider endpoints, framework version, container paths and read-only mount policy. Set each declared machine-path environment variable to its trusted prepared directory, and set the declared API-key credentials. Machine-local paths select artifacts; they do not alter experiment semantics and are not presented as a cryptographic identity scheme. +Maka benchmark subjects also freeze a versioned Hosted Execution tool profile. The +`headless-coding-v1` profile admits only `Bash`, `Read`, `Write`, `Edit`, `Glob`, `Grep`, and +`apply_patch` as agent-permission tools. Runtime-owned `ArchiveRead` remains available when context +budget pruning creates an archive placeholder. Product task, goal, scheduling, skill, interaction, +background-control, and parent-agent tools are outside this profile and must not drift into a +benchmark run. + The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains. diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json index 3906533f6f..c3ef8b0b2c 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json @@ -96,7 +96,8 @@ "thinkingLevel": "max", "permissionMode": "bypass", "collaborationMode": "agent", - "orchestrationMode": "default" + "orchestrationMode": "default", + "toolProfile": "headless-coding-v1" } }, { diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json index 8fbce280d0..f98ab8dc26 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json @@ -74,7 +74,8 @@ "permissionMode": "bypass", "collaborationMode": "agent", "orchestrationMode": "default", - "hostSettlementTimeoutMs": 120000 + "hostSettlementTimeoutMs": 120000, + "toolProfile": "headless-coding-v1" } }, { diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index a20d667e02..ebe1aba229 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -635,6 +635,7 @@ test('eight-arm spec and wrappers freeze the working provider contracts', async config: { connectionSlug?: string; baseUrl?: string; + toolProfile?: string; args?: string[]; credentialEnvironment?: Record; }; @@ -646,6 +647,7 @@ test('eight-arm spec and wrappers freeze the working provider contracts', async assert.deepEqual(maka.credentials, ['DEEPSEEK_API_KEY']); assert.equal(maka.config.connectionSlug, 'env-deepseek'); assert.equal(maka.config.baseUrl, 'https://api.deepseek.com'); + assert.equal(maka.config.toolProfile, 'headless-coding-v1'); assert.equal(codex.config.args?.includes('--ephemeral'), true); assert.equal(codex.config.args?.includes('--skip-git-repo-check'), true); assert.equal(claude.config.args?.includes('--bare'), true); @@ -766,7 +768,10 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', 'utf8', ), ) as { - subjects: Array<{ id: string; config: { args?: string[]; webTools?: string } }>; + subjects: Array<{ + id: string; + config: { args?: string[]; webTools?: string; toolProfile?: string }; + }>; execution: { maxConcurrentTaskGroups: number }; executor: { config: { mounts: Array<{ target: string }> } }; }; @@ -797,6 +802,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', (maka.config as { hostSettlementTimeoutMs?: number }).hostSettlementTimeoutMs, 120_000, ); + assert.equal(maka.config.toolProfile, 'headless-coding-v1'); assert.deepEqual( zcode.config.args?.slice( zcode.config.args.indexOf('--disallowedTools'), @@ -1005,6 +1011,7 @@ function makaConfig() { collaborationMode: 'agent', orchestrationMode: 'default', hostSettlementTimeoutMs: 120_000, + toolProfile: 'headless-coding-v1', }; } diff --git a/packages/eval/src/maka-subject.ts b/packages/eval/src/maka-subject.ts index 6440852ca6..dc0da5c621 100644 --- a/packages/eval/src/maka-subject.ts +++ b/packages/eval/src/maka-subject.ts @@ -1,6 +1,9 @@ import { randomUUID } from 'node:crypto'; -import type { HostedExecutionStartInput } from '@maka/runtime-host/protocol'; -import { decodeHostedExecutionProjection } from '@maka/runtime-host/protocol'; +import { + decodeHostedExecutionProjection, + HOSTED_EXECUTION_TOOL_PROFILES, + type HostedExecutionStartInput, +} from '@maka/runtime-host/protocol'; import type { JsonObject } from './experiment.js'; import type { NormalizedUsage } from './result.js'; import type { SubjectAdapter, SubjectExecutionContext } from './runner.js'; @@ -28,6 +31,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter { }, content: { text: context.taskInput }, maxSteps: positive(cell.budget.maxSteps, 'budget.maxSteps'), + toolProfile: config.toolProfile, }; const payload = Buffer.from( JSON.stringify({ @@ -209,6 +213,7 @@ interface MakaConfig { readonly permissionMode: HostedExecutionStartInput['session']['permissionMode']; readonly collaborationMode: HostedExecutionStartInput['session']['collaborationMode']; readonly orchestrationMode: HostedExecutionStartInput['session']['orchestrationMode']; + readonly toolProfile: NonNullable; } function decodeConfig(value: JsonObject): MakaConfig { @@ -224,6 +229,7 @@ function decodeConfig(value: JsonObject): MakaConfig { 'collaborationMode', 'orchestrationMode', 'hostSettlementTimeoutMs', + 'toolProfile', ]; if (Object.hasOwn(value, 'webTools')) fields.push('webTools'); const config = exact(value, fields); @@ -236,6 +242,9 @@ function decodeConfig(value: JsonObject): MakaConfig { config.hostSettlementTimeoutMs, 'Maka config.hostSettlementTimeoutMs', ); + if (!(HOSTED_EXECUTION_TOOL_PROFILES as readonly unknown[]).includes(config.toolProfile)) { + throw new Error('Maka config.toolProfile is invalid'); + } return { ...config, webTools, hostSettlementTimeoutMs } as unknown as MakaConfig; } diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 6d883e7470..8bfa090ab0 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -37,6 +37,7 @@ import { type ScannedSkill } from '@maka/runtime/skills'; import { agentGraphIdForRootSession } from '@maka/runtime/stream-graph-coordinator'; import { buildParentAgentTools } from '@maka/runtime/subagent-tools'; import { SESSION_RECAP_INSTRUCTION } from '@maka/runtime/session-recap'; +import { hostedExecutionToolNames } from '../server/hosted-execution-tool-profile.js'; import { createToolResultArchiveCapability } from '@maka/runtime/tool-result-archive-capability'; import { stableHash, toolCatalogHash } from '@maka/runtime/request-shape'; import { toolAvailabilityHash } from '@maka/runtime/tool-availability'; @@ -2365,6 +2366,32 @@ test('a bound tool ceiling excludes dynamic Client Capability tools', () => { ); }); +test('the headless coding profile binds the exact Eval tool ceiling', () => { + const composition = createInteractiveRunComposer({ + runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, + skills: { + readCanonicalModelInventory: async () => ({ inventory: [] }), + } as unknown as HostSkillCatalogCoordinator, + memory: {} as HostMemoryCoordinator, + taskLedger: {} as TaskLedgerStore, + builtinTools: {}, + boundToolNames: hostedExecutionToolNames('headless-coding-v1'), + parentAgentTools: buildParentAgentTools(), + scheduledTaskTool: { + name: 'ScheduledTask', + description: 'Must stay outside the Eval ceiling.', + parameters: {}, + impl: async () => 'scheduled', + }, + }); + + assert.deepEqual( + composition.tools.map(({ name }) => name), + ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'apply_patch'], + ); + assert.deepEqual(composition.toolAvailability.groups, []); +}); + function skillFixture(id: string, description: string, content: string): ScannedSkill { return { ref: `project:agents:${id}`, diff --git a/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts new file mode 100644 index 0000000000..fadb08ef0c --- /dev/null +++ b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { decodeHostedExecutionStartInput } from '../protocol/index.js'; +import { HostedExecutionToolProfileRegistry } from '../server/hosted-execution-tool-profile.js'; + +test('hosted execution tool profiles are explicit protocol inputs', () => { + const decoded = decodeHostedExecutionStartInput({ + executionId: '00000000-0000-4000-8000-000000000001', + session: { + workspace: { kind: 'host_path', path: '/workspace' }, + modelTarget: { kind: 'explicit', connectionSlug: 'provider', model: 'model' }, + }, + content: { text: 'solve' }, + toolProfile: 'headless-coding-v1', + }); + assert.equal(decoded.toolProfile, 'headless-coding-v1'); + assert.throws( + () => + decodeHostedExecutionStartInput({ + ...decoded, + toolProfile: 'unknown-profile', + }), + /Invalid Hosted execution tool profile/u, + ); +}); + +test('hosted execution tool profiles remain process-local and bounded to one execution', async () => { + const registry = new HostedExecutionToolProfileRegistry(); + assert.equal(registry.toolNamesFor('execution'), undefined); + await registry.run('execution', 'headless-coding-v1', async () => { + assert.deepEqual(registry.toolNamesFor('execution'), [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'Glob', + 'Grep', + 'apply_patch', + ]); + }); + assert.equal(registry.toolNamesFor('execution'), undefined); +}); diff --git a/packages/runtime-host/src/protocol/hosted-execution.ts b/packages/runtime-host/src/protocol/hosted-execution.ts index 79cdca59dd..c486f47442 100644 --- a/packages/runtime-host/src/protocol/hosted-execution.ts +++ b/packages/runtime-host/src/protocol/hosted-execution.ts @@ -11,6 +11,9 @@ import { defineOperation } from './operation-spec.js'; import { decodeSessionCreateInput, type SessionCreateInput } from './session-catalog.js'; import { decodeMessageContent } from './turn.js'; +export const HOSTED_EXECUTION_TOOL_PROFILES = ['headless-coding-v1'] as const; +export type HostedExecutionToolProfile = (typeof HOSTED_EXECUTION_TOOL_PROFILES)[number]; + const ERRORS = [ 'host_not_ready', 'host_draining', @@ -25,6 +28,7 @@ export interface HostedExecutionStartInput { readonly session: Omit; readonly content: MessageContent; readonly maxSteps?: number; + readonly toolProfile?: HostedExecutionToolProfile; } export interface HostedExecutionReferenceInput { @@ -95,7 +99,7 @@ export function decodeHostedExecutionStartInput(value: unknown): HostedExecution value, 'Hosted execution start input', ['executionId', 'session', 'content'], - ['maxSteps'], + ['maxSteps', 'toolProfile'], ); const executionId = requireEntityId(input.executionId, 'executionId'); const { sessionId: _sessionId, ...session } = decodeSessionCreateInput({ @@ -109,9 +113,22 @@ export function decodeHostedExecutionStartInput(value: unknown): HostedExecution ...(input.maxSteps === undefined ? {} : { maxSteps: requirePositiveCount(input.maxSteps, 'maxSteps') }), + ...(input.toolProfile === undefined + ? {} + : { toolProfile: decodeHostedExecutionToolProfile(input.toolProfile) }), }; } +function decodeHostedExecutionToolProfile(value: unknown): HostedExecutionToolProfile { + if ( + typeof value !== 'string' || + !(HOSTED_EXECUTION_TOOL_PROFILES as readonly string[]).includes(value) + ) { + throw invalidProtocolFrame('Invalid Hosted execution tool profile'); + } + return value as HostedExecutionToolProfile; +} + export function decodeHostedExecutionReferenceInput(value: unknown): HostedExecutionReferenceInput { const input = requireExactRecord(value, 'Hosted execution reference', ['executionId']); return { executionId: requireEntityId(input.executionId, 'executionId') }; diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index e3e47288e5..77876ee6a4 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -128,6 +128,7 @@ import { import { HostInteractionCoordinator } from './interaction-coordinator.js'; import { HostInteractiveTurnCoordinator } from './interactive-turn-coordinator.js'; import { ensureBootstrapRuntimePolicy } from './bootstrap-runtime-policy.js'; +import { HostedExecutionToolProfileRegistry } from './hosted-execution-tool-profile.js'; import { HostMemoryCoordinator } from './memory-coordinator.js'; import { HostMemoryExtractionCoordinator } from './memory-extraction-coordinator.js'; import { MemoryExtractionSessionLane } from './memory-extraction-session-lane.js'; @@ -277,6 +278,7 @@ export async function createExecutionRuntimeHostComposition( }); await stores.messageReceiptStore.beginHostEpoch(context.hostEpoch); const backends = new BackendRegistry(); + const hostedExecutionToolProfiles = new HostedExecutionToolProfileRegistry(); backends.register('fake', (backendContext) => new FakeBackend(backendContext)); const runtimePolicyActivation = new RuntimePolicyActivationGate(); const runtimePolicy = new HostRuntimePolicyCoordinator( @@ -643,6 +645,8 @@ export async function createExecutionRuntimeHostComposition( parentAgentTools: childAgentTools.parentTools, childTools: childAgentTools.childTools, worktreePatchWriteBackAvailable: true, + resolveBoundToolNames: (sessionId) => + hostedExecutionToolProfiles.toolNamesFor(sessionId), }), memoryExtraction, artifacts: openedArtifactStore, @@ -1239,7 +1243,10 @@ export async function createExecutionRuntimeHostComposition( }, }); const hostedExecutions = new HostHostedExecutionCoordinator( - (input, signal) => hostedExecutionRunner.run(input, signal), + (input, signal) => + hostedExecutionToolProfiles.run(input.executionId, input.toolProfile, () => + hostedExecutionRunner.run(input, signal), + ), context.requestDrain, ); let recoverySessions: Awaited> = []; diff --git a/packages/runtime-host/src/server/hosted-execution-tool-profile.ts b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts new file mode 100644 index 0000000000..2e7c8284d3 --- /dev/null +++ b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts @@ -0,0 +1,43 @@ +import type { HostedExecutionToolProfile } from '../protocol/index.js'; + +const HEADLESS_CODING_V1_TOOL_NAMES = [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'Glob', + 'Grep', + 'apply_patch', +] as const; + +export function hostedExecutionToolNames(profile: HostedExecutionToolProfile): readonly string[] { + if (profile === 'headless-coding-v1') return HEADLESS_CODING_V1_TOOL_NAMES; + profile satisfies never; + throw new Error('Unknown Hosted execution tool profile'); +} + +export class HostedExecutionToolProfileRegistry { + readonly #profiles = new Map(); + + async run( + executionId: string, + profile: HostedExecutionToolProfile | undefined, + operation: () => Promise, + ): Promise { + if (!profile) return await operation(); + if (this.#profiles.has(executionId)) { + throw new Error('Hosted execution tool profile is already registered'); + } + this.#profiles.set(executionId, profile); + try { + return await operation(); + } finally { + this.#profiles.delete(executionId); + } + } + + toolNamesFor(executionId: string): readonly string[] | undefined { + const profile = this.#profiles.get(executionId); + return profile ? hostedExecutionToolNames(profile) : undefined; + } +} diff --git a/packages/runtime-host/src/server/interactive-run-composer.ts b/packages/runtime-host/src/server/interactive-run-composer.ts index 5bc322b5f3..936e67b732 100644 --- a/packages/runtime-host/src/server/interactive-run-composer.ts +++ b/packages/runtime-host/src/server/interactive-run-composer.ts @@ -86,6 +86,7 @@ export interface InteractiveRunComposerInput { readonly childInstruction?: string; readonly sideConversation?: boolean; readonly boundTools?: readonly MakaTool[]; + readonly boundToolNames?: readonly string[]; readonly skillBudget?: SkillCatalogBudgetOptions; readonly platform?: NodeJS.Platform; readonly shell?: string; @@ -112,6 +113,9 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) const inventorySnapshotFor = createTurnSkillInventorySnapshotResolver(input.skills); const inventoryFor: SkillInventoryResolver = async (context) => (await inventorySnapshotFor(context)).inventory; + if (input.boundTools && input.boundToolNames) { + throw new Error('Interactive tool bindings are ambiguous'); + } const defaultTools = input.boundTools ? input.boundTools : buildDefaultHostTools( @@ -125,11 +129,15 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) input.plan, input.deepResearch?.tools, ); - const clientCapabilityTools = input.boundTools ? [] : (input.clientCapabilities?.tools ?? []); + const clientCapabilityTools = + input.boundTools || input.boundToolNames ? [] : (input.clientCapabilities?.tools ?? []); const unscopedCandidateTools = [...defaultTools, ...clientCapabilityTools]; - const candidateTools = input.deepResearch + const routedCandidateTools = input.deepResearch ? unscopedCandidateTools.filter(isDeepResearchToolAllowed) : unscopedCandidateTools; + const candidateTools = input.boundToolNames + ? bindToolsByName(routedCandidateTools, input.boundToolNames) + : routedCandidateTools; const activeExecution = input.plan ? activePlanExecution(input.plan.state) : undefined; const selectedTools = input.plan ? selectCollaborationTools({ @@ -142,7 +150,10 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) const productSurface = projectEffectiveProductToolSurface({ host: 'runtime-host', tools: selectedTools, - policy: { economy: !process.env.MAKA_DISABLE_DEFERRED_TOOLS }, + policy: { + economy: + input.boundTools || input.boundToolNames ? false : !process.env.MAKA_DISABLE_DEFERRED_TOOLS, + }, }); // A bound tool list is an exact child/local activation ceiling. Dynamic // capabilities must be included by the authority that constructs that list. @@ -266,10 +277,26 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) }); } +function bindToolsByName( + tools: readonly MakaTool[], + names: readonly string[], +): readonly MakaTool[] { + if (new Set(names).size !== names.length) { + throw new Error('Hosted tool profile contains duplicate tool names'); + } + const byName = new Map(tools.map((tool) => [tool.name, tool])); + const selected = names.map((name) => byName.get(name)); + const missing = names.filter((_name, index) => selected[index] === undefined); + if (missing.length > 0) { + throw new Error(`Hosted tool profile is unavailable: ${missing.join(', ')}`); + } + return selected as MakaTool[]; +} + export interface InteractiveRunComposerFactoryInput extends Omit< InteractiveRunComposerInput, - 'runtimePolicy' | 'boundTools' | 'clientCapabilities' | 'plan' + 'runtimePolicy' | 'boundTools' | 'boundToolNames' | 'clientCapabilities' | 'plan' > { readonly clientCapabilities: HostClientCapabilityCoordinator; readonly resolveRootTools?: (sessionId: string) => Promise; @@ -277,6 +304,7 @@ export interface InteractiveRunComposerFactoryInput readonly worktreePatchWriteBackAvailable?: boolean; readonly planStore?: PlanStore; readonly deepResearchTools?: readonly MakaTool[]; + readonly resolveBoundToolNames?: (sessionId: string) => readonly string[] | undefined; } export function createInteractiveRunComposerFactory( @@ -341,6 +369,9 @@ export function createInteractiveRunComposerFactory( ? { sideConversation: true } : {}), ...(boundTools ? { boundTools } : {}), + ...(!boundTools && input.resolveBoundToolNames + ? { boundToolNames: input.resolveBoundToolNames(backendContext.sessionId) } + : {}), ...(clientCapabilities ? { clientCapabilities } : {}), ...(input.builtinTools ? { builtinTools: input.builtinTools } : {}), ...(hostTools.length > 0 ? { hostTools } : {}), From b718bc0d95c48b19abb54a48b50c5cbec8700093 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 06:41:11 +0000 Subject: [PATCH 02/13] fix(eval): enforce the complete headless tool surface --- packages/eval/README.md | 9 ++++--- .../hosted-execution-tool-profile.test.ts | 3 +++ .../src/server/execution-composition.ts | 4 ++- .../server/hosted-execution-tool-profile.ts | 25 +++++++++++++++++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/packages/eval/README.md b/packages/eval/README.md index 7ebbed3ed3..d6c1131b7a 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -28,9 +28,10 @@ The checked-in Terminal-Bench 2.1 four-arm cohort is `experiments/terminal-bench Maka benchmark subjects also freeze a versioned Hosted Execution tool profile. The `headless-coding-v1` profile admits only `Bash`, `Read`, `Write`, `Edit`, `Glob`, `Grep`, and -`apply_patch` as agent-permission tools. Runtime-owned `ArchiveRead` remains available when context -budget pruning creates an archive placeholder. Product task, goal, scheduling, skill, interaction, -background-control, and parent-agent tools are outside this profile and must not drift into a -benchmark run. +`apply_patch` as agent-permission tool candidates. Provider-specific routing remains authoritative: +DeepSeek Responses exposes `apply_patch` instead of `Write` and `Edit`. Runtime-owned `ArchiveRead` +remains available when context budget pruning creates an archive placeholder. Long-term-memory +triggers and product task, goal, scheduling, skill, interaction, background-control, and parent-agent +tools are outside this profile and must not drift into a benchmark run. The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains. diff --git a/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts index fadb08ef0c..3ea4173c67 100644 --- a/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts +++ b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts @@ -27,6 +27,7 @@ test('hosted execution tool profiles are explicit protocol inputs', () => { test('hosted execution tool profiles remain process-local and bounded to one execution', async () => { const registry = new HostedExecutionToolProfileRegistry(); assert.equal(registry.toolNamesFor('execution'), undefined); + assert.equal(registry.allowsMemoryExtractionFor('execution'), true); await registry.run('execution', 'headless-coding-v1', async () => { assert.deepEqual(registry.toolNamesFor('execution'), [ 'Bash', @@ -37,6 +38,8 @@ test('hosted execution tool profiles remain process-local and bounded to one exe 'Grep', 'apply_patch', ]); + assert.equal(registry.allowsMemoryExtractionFor('execution'), false); }); assert.equal(registry.toolNamesFor('execution'), undefined); + assert.equal(registry.allowsMemoryExtractionFor('execution'), true); }); diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 77876ee6a4..4a15f7d2fa 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -648,7 +648,9 @@ export async function createExecutionRuntimeHostComposition( resolveBoundToolNames: (sessionId) => hostedExecutionToolProfiles.toolNamesFor(sessionId), }), - memoryExtraction, + ...(hostedExecutionToolProfiles.allowsMemoryExtractionFor(backendContext.sessionId) + ? { memoryExtraction } + : {}), artifacts: openedArtifactStore, executionArtifacts, usage: openedUsageStores, diff --git a/packages/runtime-host/src/server/hosted-execution-tool-profile.ts b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts index 2e7c8284d3..3618663b67 100644 --- a/packages/runtime-host/src/server/hosted-execution-tool-profile.ts +++ b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts @@ -10,12 +10,28 @@ const HEADLESS_CODING_V1_TOOL_NAMES = [ 'apply_patch', ] as const; -export function hostedExecutionToolNames(profile: HostedExecutionToolProfile): readonly string[] { - if (profile === 'headless-coding-v1') return HEADLESS_CODING_V1_TOOL_NAMES; +interface HostedExecutionToolProfilePolicy { + readonly toolNames: readonly string[]; + readonly memoryExtraction: boolean; +} + +function hostedExecutionToolProfilePolicy( + profile: HostedExecutionToolProfile, +): HostedExecutionToolProfilePolicy { + if (profile === 'headless-coding-v1') { + return { + toolNames: HEADLESS_CODING_V1_TOOL_NAMES, + memoryExtraction: false, + }; + } profile satisfies never; throw new Error('Unknown Hosted execution tool profile'); } +export function hostedExecutionToolNames(profile: HostedExecutionToolProfile): readonly string[] { + return hostedExecutionToolProfilePolicy(profile).toolNames; +} + export class HostedExecutionToolProfileRegistry { readonly #profiles = new Map(); @@ -40,4 +56,9 @@ export class HostedExecutionToolProfileRegistry { const profile = this.#profiles.get(executionId); return profile ? hostedExecutionToolNames(profile) : undefined; } + + allowsMemoryExtractionFor(executionId: string): boolean { + const profile = this.#profiles.get(executionId); + return profile ? hostedExecutionToolProfilePolicy(profile).memoryExtraction : true; + } } From 8302c2efdb6e5a125ea471aaf1071d41969f5982 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 06:59:39 +0000 Subject: [PATCH 03/13] fix(eval): remove web tools and filter benchmark egress --- packages/eval/README.md | 9 +++ ...bench-2.1-deepseek-v4-flash-eight-arm.json | 13 +++- ...-bench-2.1-deepseek-v4-flash-four-arm.json | 15 ++++ packages/eval/harbor/egress_filter.py | 48 ++++++++++++ packages/eval/harbor/test_egress_filter.py | 35 +++++++++ packages/eval/package.json | 2 +- .../__tests__/lifecycle-boundaries.test.ts | 19 ++++- .../provider-web-tool-surface.test.ts | 39 ++++++++++ packages/eval/src/harbor-external-subject.ts | 16 +++- packages/eval/src/harness-executor.ts | 73 ++++++++++++++++++- .../eval/src/provider-web-tool-surface.ts | 40 ++++++++++ 11 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 packages/eval/harbor/egress_filter.py create mode 100644 packages/eval/harbor/test_egress_filter.py create mode 100644 packages/eval/src/__tests__/provider-web-tool-surface.test.ts create mode 100644 packages/eval/src/provider-web-tool-surface.ts diff --git a/packages/eval/README.md b/packages/eval/README.md index d6c1131b7a..098fd26964 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -34,4 +34,13 @@ remains available when context budget pruning creates an archive placeholder. Lo triggers and product task, goal, scheduling, skill, interaction, background-control, and parent-agent tools are outside this profile and must not drift into a benchmark run. +Every benchmark subject removes `WebSearch`, `WebFetch`, and `FetchURL` from the provider-visible +tool list. Maka enforces that through its Hosted Execution profile; external harnesses pass through +the Eval metering proxy, which structurally removes named and provider-native web tools from JSON +requests. Shell networking remains enabled. The configured HTTPS egress proxy blocks only +Terminal-Bench repository paths for `harbor-framework/terminal-bench*` and +`NousResearch/terminal-bench*` on GitHub, the GitHub repositories API, raw content, and codeload. +The machine must provide `MAKA_EVAL_EGRESS_PROXY_URL` and a trusted CA file through +`MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH`. + The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains. diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json index c3ef8b0b2c..7338980cea 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json @@ -29,7 +29,17 @@ "no_proxy", "SSH_AUTH_SOCK" ], + "egressProxy": { + "urlEnv": "MAKA_EVAL_EGRESS_PROXY_URL", + "caCertificatePathEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" + }, "mounts": [ + { + "sourceEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "target": "/opt/maka-egress/mitmproxy-ca-cert.pem", + "readOnly": true + }, { "sourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "target": "/opt/maka-agent", @@ -147,6 +157,7 @@ "--disallowedTools", "WebSearch", "WebFetch", + "FetchURL", "--dangerously-skip-permissions", "--model", "deepseek-v4-flash", @@ -245,7 +256,7 @@ "--cwd", "{{task.cwd}}", "--disallowedTools", - "WebSearch,WebFetch", + "WebSearch,WebFetch,FetchURL", "--prompt", "{{task.input}}" ] diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json index f98ab8dc26..a70f257d8f 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json @@ -28,7 +28,17 @@ "https_proxy", "no_proxy" ], + "egressProxy": { + "urlEnv": "MAKA_EVAL_EGRESS_PROXY_URL", + "caCertificatePathEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" + }, "mounts": [ + { + "sourceEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "target": "/opt/maka-egress/mitmproxy-ca-cert.pem", + "readOnly": true + }, { "sourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "target": "/opt/maka-agent", @@ -69,6 +79,7 @@ "runtimeHostsPath": "/tmp/maka-runtime-hosts", "connectionSlug": "env-openai", "baseUrl": "https://api.deepseek.com", + "webTools": "disabled", "model": "deepseek-v4-flash", "thinkingLevel": "max", "permissionMode": "bypass", @@ -114,6 +125,10 @@ "--print", "--output-format", "json", + "--disallowedTools", + "WebSearch", + "WebFetch", + "FetchURL", "--dangerously-skip-permissions", "--model", "deepseek-v4-flash", diff --git a/packages/eval/harbor/egress_filter.py b/packages/eval/harbor/egress_filter.py new file mode 100644 index 0000000000..941996b088 --- /dev/null +++ b/packages/eval/harbor/egress_filter.py @@ -0,0 +1,48 @@ +"""Path-level egress filter for Terminal-Bench evaluation subjects.""" + +from __future__ import annotations + +from urllib.parse import unquote, urlsplit + +BLOCKED_OWNERS = {"harbor-framework", "nousresearch"} + + +def blocked_terminal_bench_url(raw_url: str) -> bool: + try: + url = urlsplit(raw_url) + except ValueError: + return False + host = (url.hostname or "").lower().rstrip(".") + segments = [unquote(part).lower() for part in url.path.split("/") if part] + if host == "api.github.com": + return ( + len(segments) >= 3 + and segments[0] == "repos" + and blocked_repository(segments[1], segments[2]) + ) + if host in {"github.com", "raw.githubusercontent.com", "codeload.github.com"}: + return len(segments) >= 2 and blocked_repository(segments[0], segments[1]) + return False + + +def blocked_repository(owner: str, repository: str) -> bool: + return owner.lower() in BLOCKED_OWNERS and repository.lower().startswith("terminal-bench") + + +try: + from mitmproxy import http +except ImportError: + http = None + + +def request(flow: object) -> None: + if http is None: + raise RuntimeError("mitmproxy is required to run the Eval egress filter") + request_url = flow.request.pretty_url + if not blocked_terminal_bench_url(request_url): + return + flow.response = http.Response.make( + 451, + b"Terminal-Bench benchmark source access is blocked during evaluation.\n", + {"Content-Type": "text/plain; charset=utf-8"}, + ) diff --git a/packages/eval/harbor/test_egress_filter.py b/packages/eval/harbor/test_egress_filter.py new file mode 100644 index 0000000000..e85cfc1426 --- /dev/null +++ b/packages/eval/harbor/test_egress_filter.py @@ -0,0 +1,35 @@ +import importlib.util +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).with_name("egress_filter.py") +SPEC = importlib.util.spec_from_file_location("maka_eval_egress_filter", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class EgressFilterTest(unittest.TestCase): + def test_blocks_only_terminal_bench_repository_paths(self) -> None: + blocked = [ + "https://github.com/harbor-framework/terminal-bench-2", + "https://github.com/NousResearch/terminal-bench/tree/main", + "https://api.github.com/repos/harbor-framework/terminal-bench-2-1/contents/tasks", + "https://raw.githubusercontent.com/NousResearch/terminal-bench/main/README.md", + "https://codeload.github.com/harbor-framework/terminal-bench-2.1/tar.gz/main", + ] + allowed = [ + "https://github.com/harbor-framework/harbor", + "https://api.github.com/repos/other/terminal-bench", + "https://raw.githubusercontent.com/other/project/main/data.txt", + "https://pypi.org/simple/requests/", + "https://huggingface.co/datasets/example/data", + ] + for url in blocked: + self.assertTrue(MODULE.blocked_terminal_bench_url(url), url) + for url in allowed: + self.assertFalse(MODULE.blocked_terminal_bench_url(url), url) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/eval/package.json b/packages/eval/package.json index deb9ba3c9d..3c9a52791d 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -19,7 +19,7 @@ "clean": "node ../../scripts/clean-paths.mjs dist tsconfig.tsbuildinfo", "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py" + "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py && python3 harbor/test_egress_filter.py" }, "dependencies": { "@maka/core": "0.1.0", diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index ebe1aba229..295b83bc54 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -773,16 +773,31 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', config: { args?: string[]; webTools?: string; toolProfile?: string }; }>; execution: { maxConcurrentTaskGroups: number }; - executor: { config: { mounts: Array<{ target: string }> } }; + executor: { + config: { + mounts: Array<{ target: string }>; + egressProxy: { + urlEnv: string; + caCertificatePathEnv: string; + containerCaPath: string; + }; + }; + }; }; assert.equal(spec.execution.maxConcurrentTaskGroups, 16); assert.deepEqual( spec.subjects.map(({ id }) => id), ['maka', 'codex', 'claude-code', 'reasonix', 'opencode', 'kimi-code', 'zcode', 'pi'], ); + assert.deepEqual(spec.executor.config.egressProxy, { + urlEnv: 'MAKA_EVAL_EGRESS_PROXY_URL', + caCertificatePathEnv: 'MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH', + containerCaPath: '/opt/maka-egress/mitmproxy-ca-cert.pem', + }); assert.deepEqual( spec.executor.config.mounts.map(({ target }) => target), [ + '/opt/maka-egress/mitmproxy-ca-cert.pem', '/opt/maka-agent', '/opt/maka-node-toolchain', '/opt/maka-codex-toolchain', @@ -808,7 +823,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', zcode.config.args.indexOf('--disallowedTools'), zcode.config.args.indexOf('--disallowedTools') + 2, ), - ['--disallowedTools', 'WebSearch,WebFetch'], + ['--disallowedTools', 'WebSearch,WebFetch,FetchURL'], ); assert.equal(pi.config.args?.includes('/opt/maka-pi-toolchain/bin/pi'), true); assert.equal(pi.config.args?.includes('--mode'), true); diff --git a/packages/eval/src/__tests__/provider-web-tool-surface.test.ts b/packages/eval/src/__tests__/provider-web-tool-surface.test.ts new file mode 100644 index 0000000000..92c72f8187 --- /dev/null +++ b/packages/eval/src/__tests__/provider-web-tool-surface.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { removeEvalWebTools } from '../provider-web-tool-surface.js'; + +test('Eval removes named and provider-native web tools from provider requests', () => { + const projected = removeEvalWebTools( + Buffer.from( + JSON.stringify({ + model: 'deepseek-v4-flash', + tools: [ + { type: 'function', function: { name: 'Read' } }, + { type: 'function', function: { name: 'WebSearch' } }, + { name: 'WebFetch', input_schema: {} }, + { type: 'custom', name: 'FetchURL' }, + { type: 'web_search_20250305' }, + { type: 'web_fetch_preview' }, + ], + }), + ), + ); + + assert.equal(projected.removed, 5); + assert.deepEqual(JSON.parse(projected.body.toString('utf8')), { + model: 'deepseek-v4-flash', + tools: [{ type: 'function', function: { name: 'Read' } }], + }); +}); + +test('Eval preserves non-JSON and requests without web tools byte-for-byte', () => { + for (const body of [ + Buffer.from('not-json'), + Buffer.from(JSON.stringify({ model: 'deepseek-v4-flash' })), + Buffer.from(JSON.stringify({ tools: [{ type: 'function', function: { name: 'Read' } }] })), + ]) { + const projected = removeEvalWebTools(body); + assert.equal(projected.removed, 0); + assert.equal(projected.body, body); + } +}); diff --git a/packages/eval/src/harbor-external-subject.ts b/packages/eval/src/harbor-external-subject.ts index 66bd8a3708..925f883b28 100644 --- a/packages/eval/src/harbor-external-subject.ts +++ b/packages/eval/src/harbor-external-subject.ts @@ -10,6 +10,7 @@ import { type ExternalProfile as Profile, } from './toolchain-verification.js'; import { isInferenceAdmissionEvent } from './provider-admission.js'; +import { removeEvalWebTools } from './provider-web-tool-surface.js'; import { takeRelayResultToken, writeRelayResult } from './relay-result-frame.js'; const resultToken = takeRelayResultToken(); @@ -91,6 +92,7 @@ try { requests: proxy.requestCount(), admittedRequests: proxy.admittedRequestCount(), usageRequests: proxy.usageRequestCount(), + removedWebTools: proxy.removedWebToolCount(), }); artifacts.push( { kind: 'external-process', profile, exitCode: result.exitCode }, @@ -103,6 +105,7 @@ try { admittedRequests: proxy.admittedRequestCount(), usageRequests: proxy.usageRequestCount(), usageComplete: proxy.usageComplete(), + removedWebTools: proxy.removedWebToolCount(), }, fileArtifact('wrapper-state', statePath, profile), ); @@ -181,7 +184,7 @@ async function prepareProfile( await writePolicy( rooted(root, '/etc/claude-code'), 'managed-settings.json', - '{"permissions":{"deny":["WebSearch","WebFetch"]}}\n', + '{"permissions":{"deny":["WebSearch","WebFetch","FetchURL"]}}\n', ); } else if (selected === 'reasonix') { const modelReference = executableArgs[executableArgs.indexOf('--model') + 1]; @@ -268,6 +271,7 @@ async function prepareProfile( '[permission]', 'rules = [', ' { decision = "deny", scope = "user", pattern = "WebSearch", reason = "Disabled for eval parity" },', + ' { decision = "deny", scope = "user", pattern = "WebFetch", reason = "Disabled for eval parity" },', ' { decision = "deny", scope = "user", pattern = "FetchURL", reason = "Disabled for eval parity" },', ']', '', @@ -370,7 +374,7 @@ async function prepareProfile( model: 'deepseek/deepseek-v4-flash', permission: { mode: 'yolo', - disallowedTools: ['WebSearch', 'WebFetch'], + disallowedTools: ['WebSearch', 'WebFetch', 'FetchURL'], }, features: { memory: false, mcp: false }, plugins: { enabled: false }, @@ -587,6 +591,7 @@ async function startMeteringProxy( admittedRequestCount(): number; usageRequestCount(): number; usageComplete(): boolean; + removedWebToolCount(): number; close(): Promise; }> { const total = zeroUsage(); @@ -594,11 +599,13 @@ async function startMeteringProxy( let requests = 0; let admittedRequests = 0; let usageRequests = 0; + let removedWebTools = 0; const active = new Set>(); const server = createServer((request, response) => { const operation = (async () => { requests += 1; - const body = await readRequest(request); + const projected = removeEvalWebTools(await readRequest(request)); + removedWebTools += projected.removed; const target = joinUpstream(upstreamBaseUrl, request.url ?? '/'); const headers = new Headers(); for (const [name, value] of Object.entries(request.headers)) { @@ -616,7 +623,7 @@ async function startMeteringProxy( const upstream = await fetch(target, { method: request.method, headers, - body: body.length === 0 ? undefined : new Uint8Array(body), + body: projected.body.length === 0 ? undefined : new Uint8Array(projected.body), }); response.statusCode = upstream.status; upstream.headers.forEach((value, name) => { @@ -666,6 +673,7 @@ async function startMeteringProxy( admittedRequestCount: () => admittedRequests, usageRequestCount: () => usageRequests, usageComplete: () => admittedRequests > 0 && usageRequests === admittedRequests, + removedWebToolCount: () => removedWebTools, close: async () => { await Promise.allSettled([...active]); await closeServer(server); diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index 85c3eaadf5..1ee4b94401 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -47,6 +47,7 @@ interface RelayState { readonly taskInput: string; readonly credentials: Readonly>; readonly cwd: string; + readonly executionEnvironment: Readonly>; used: boolean; diagnostic?: SubjectProcessDiagnostic; } @@ -240,7 +241,10 @@ function relayContext(state: RelayState, signal?: AbortSignal): SubjectExecution kind: 'execute', command: input.command, args: input.args, - environment: input.environment ?? {}, + environment: mergeExecutionEnvironment( + state.executionEnvironment, + input.environment ?? {}, + ), credentials, resultToken, captureStdout: input.captureStdout ?? true, @@ -343,6 +347,7 @@ async function startTrial( const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier'); const environmentConfig = { ...options.environment, mounts: resolveMounts(options.mounts) }; const relayPath = resolve(dirname(fileURLToPath(import.meta.url)), '../harbor'); + const executionEnvironment = egressExecutionEnvironment(options.egressProxy); const environment = preparationEnvironment( framework, relayPath, @@ -449,6 +454,7 @@ async function startTrial( taskInput: ready.instruction, credentials, cwd: ready.cwd, + executionEnvironment, used: false, }, }; @@ -596,6 +602,42 @@ function preparationEnvironment( }; } +function mergeExecutionEnvironment( + required: Readonly>, + subject: Readonly>, +): Record { + const overlap = Object.keys(required).filter((name) => Object.hasOwn(subject, name)); + if (overlap.length > 0) { + throw new Error(`subject environment overrides Eval egress policy: ${overlap.join(', ')}`); + } + return { ...subject, ...required }; +} + +function egressExecutionEnvironment( + options: HarnessOptions['egressProxy'], +): Readonly> { + if (!options) return {}; + const proxyUrl = process.env[options.urlEnv]!; + if (!URL.canParse(proxyUrl) || new URL(proxyUrl).protocol !== 'http:') { + throw new Error(`machine path ${options.urlEnv} must be an HTTP proxy URL`); + } + const proxyHost = new URL(proxyUrl).hostname; + const noProxy = `127.0.0.1,localhost,${proxyHost}`; + return { + HTTP_PROXY: proxyUrl, + HTTPS_PROXY: proxyUrl, + http_proxy: proxyUrl, + https_proxy: proxyUrl, + NO_PROXY: noProxy, + no_proxy: noProxy, + SSL_CERT_FILE: options.containerCaPath, + REQUESTS_CA_BUNDLE: options.containerCaPath, + CURL_CA_BUNDLE: options.containerCaPath, + GIT_SSL_CAINFO: options.containerCaPath, + NODE_EXTRA_CA_CERTS: options.containerCaPath, + }; +} + function safeErrorCode(error: unknown): string | null { const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined; return code && ['ENOENT', 'EACCES', 'EPERM'].includes(code) ? code : null; @@ -638,6 +680,11 @@ interface HarnessOptions { readonly tasksRootEnv?: string; readonly environment: JsonObject; readonly preparationEnvironment: readonly string[]; + readonly egressProxy?: { + readonly urlEnv: string; + readonly caCertificatePathEnv: string; + readonly containerCaPath: string; + }; readonly mounts: readonly { readonly sourceEnv: string; readonly target: string; @@ -657,6 +704,7 @@ function decodeOptions(value: JsonObject, framework: Framework): HarnessOptions 'preparationEnvironment', 'mounts', ]; + if (Object.hasOwn(value, 'egressProxy')) fields.push('egressProxy'); if (framework === 'pier') fields.push('tasksRootEnv'); const options = exact(value, fields, 'executor.config'); const preparationEnvironment = array( @@ -675,17 +723,38 @@ function decodeOptions(value: JsonObject, framework: Framework): HarnessOptions trialsRootEnv: machinePathEnv(options.trialsRootEnv, 'trialsRootEnv'), environment: decodeJsonObject(options.environment, 'environment'), preparationEnvironment, + ...(Object.hasOwn(options, 'egressProxy') + ? { egressProxy: decodeEgressProxy(options.egressProxy) } + : {}), mounts: array(options.mounts, 'mounts').map((mount, index) => decodeMount(mount, index)), ...(framework === 'pier' ? { tasksRootEnv: machinePathEnv(options.tasksRootEnv, 'tasksRootEnv') } : {}), }; - for (const name of [decoded.pythonPathEnv, decoded.trialsRootEnv, decoded.tasksRootEnv]) { + for (const name of [ + decoded.pythonPathEnv, + decoded.trialsRootEnv, + decoded.tasksRootEnv, + decoded.egressProxy?.urlEnv, + decoded.egressProxy?.caCertificatePathEnv, + ]) { if (name && !process.env[name]) throw new Error(`machine path ${name} is unavailable`); } return decoded; } +function decodeEgressProxy(value: unknown): NonNullable { + const proxy = exact(value, ['urlEnv', 'caCertificatePathEnv', 'containerCaPath'], 'egressProxy'); + return { + urlEnv: machinePathEnv(proxy.urlEnv, 'egressProxy.urlEnv'), + caCertificatePathEnv: machinePathEnv( + proxy.caCertificatePathEnv, + 'egressProxy.caCertificatePathEnv', + ), + containerCaPath: absolute(proxy.containerCaPath, 'egressProxy.containerCaPath'), + }; +} + function decodeMount(value: unknown, index: number) { const mount = exact(value, ['sourceEnv', 'target', 'readOnly'], `mounts[${index}]`); const sourceEnv = machinePathEnv(mount.sourceEnv, `mounts[${index}].sourceEnv`); diff --git a/packages/eval/src/provider-web-tool-surface.ts b/packages/eval/src/provider-web-tool-surface.ts new file mode 100644 index 0000000000..bbc11ce7be --- /dev/null +++ b/packages/eval/src/provider-web-tool-surface.ts @@ -0,0 +1,40 @@ +const DISALLOWED_WEB_TOOL_NAMES = new Set(['websearch', 'webfetch', 'fetchurl']); + +export function removeEvalWebTools(body: Buffer): { + readonly body: Buffer; + readonly removed: number; +} { + if (body.length === 0) return { body, removed: 0 }; + let value: unknown; + try { + value = JSON.parse(body.toString('utf8')); + } catch { + return { body, removed: 0 }; + } + if (!isRecord(value) || !Array.isArray(value.tools)) return { body, removed: 0 }; + const tools = value.tools.filter((tool) => !isDisallowedWebTool(tool)); + const removed = value.tools.length - tools.length; + if (removed === 0) return { body, removed: 0 }; + return { body: Buffer.from(JSON.stringify({ ...value, tools }), 'utf8'), removed }; +} + +function isDisallowedWebTool(tool: unknown): boolean { + if (!isRecord(tool)) return false; + const names = [tool.name, isRecord(tool.function) ? tool.function.name : undefined]; + if ( + names.some((name) => typeof name === 'string' && DISALLOWED_WEB_TOOL_NAMES.has(normalize(name))) + ) { + return true; + } + if (typeof tool.type !== 'string') return false; + const type = normalize(tool.type); + return [...DISALLOWED_WEB_TOOL_NAMES].some((name) => type === name || type.startsWith(name)); +} + +function normalize(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/gu, ''); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} From 63ad90ae54dc9ec3f579daa1208dafd7d3ad01da Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:01:06 +0000 Subject: [PATCH 04/13] fix(eval): route Node shell traffic through egress --- packages/eval/src/harness-executor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index 1ee4b94401..fbb43752cf 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -635,6 +635,7 @@ function egressExecutionEnvironment( CURL_CA_BUNDLE: options.containerCaPath, GIT_SSL_CAINFO: options.containerCaPath, NODE_EXTRA_CA_CERTS: options.containerCaPath, + NODE_OPTIONS: '--use-env-proxy', }; } From 51d54826e83015db24ee611dbfd0a3678b7f3c94 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:06:53 +0000 Subject: [PATCH 05/13] fix(eval): preserve local harness provider proxies --- packages/eval/src/harness-executor.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index fbb43752cf..1ee4b94401 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -635,7 +635,6 @@ function egressExecutionEnvironment( CURL_CA_BUNDLE: options.containerCaPath, GIT_SSL_CAINFO: options.containerCaPath, NODE_EXTRA_CA_CERTS: options.containerCaPath, - NODE_OPTIONS: '--use-env-proxy', }; } From bc1fc0acc20431d47305f57c53770b57d6b1fe21 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:26:08 +0000 Subject: [PATCH 06/13] fix(eval): enforce subject egress contamination policy --- package-lock.json | 12 +- packages/eval/README.md | 14 +- ...bench-2.1-deepseek-v4-flash-eight-arm.json | 11 +- ...-bench-2.1-deepseek-v4-flash-four-arm.json | 11 +- .../harbor/docker-compose-egress-proxy.yaml | 26 ++++ packages/eval/harbor/egress-proxy/Dockerfile | 8 + .../eval/harbor/egress-proxy/entrypoint.sh | 13 ++ packages/eval/harbor/egress_filter.py | 146 +++++++++++++++--- packages/eval/harbor/run_trial.py | 10 ++ packages/eval/harbor/test_egress_filter.py | 68 ++++++-- packages/eval/harbor/test_run_trial_policy.py | 30 ++++ packages/eval/package.json | 5 +- .../src/__tests__/external-subject.test.ts | 3 +- .../__tests__/lifecycle-boundaries.test.ts | 28 +++- packages/eval/src/harbor-external-subject.ts | 6 +- packages/eval/src/harbor-maka-subject.ts | 2 +- packages/eval/src/harness-executor.ts | 104 ++++++++++--- packages/eval/src/maka-runtime-policy.ts | 14 +- 18 files changed, 418 insertions(+), 93 deletions(-) create mode 100644 packages/eval/harbor/docker-compose-egress-proxy.yaml create mode 100644 packages/eval/harbor/egress-proxy/Dockerfile create mode 100755 packages/eval/harbor/egress-proxy/entrypoint.sh create mode 100644 packages/eval/harbor/test_run_trial_policy.py diff --git a/package-lock.json b/package-lock.json index 3d39059b90..606425f67c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13638,7 +13638,17 @@ "license": "Apache-2.0", "dependencies": { "@maka/core": "0.1.0", - "@maka/runtime-host": "0.1.0" + "@maka/runtime-host": "0.1.0", + "undici": "^8.7.0" + } + }, + "packages/eval/node_modules/undici": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.0.tgz", + "integrity": "sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" } }, "packages/mcp": { diff --git a/packages/eval/README.md b/packages/eval/README.md index 098fd26964..0d75bd256c 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -38,9 +38,15 @@ Every benchmark subject removes `WebSearch`, `WebFetch`, and `FetchURL` from the tool list. Maka enforces that through its Hosted Execution profile; external harnesses pass through the Eval metering proxy, which structurally removes named and provider-native web tools from JSON requests. Shell networking remains enabled. The configured HTTPS egress proxy blocks only -Terminal-Bench repository paths for `harbor-framework/terminal-bench*` and -`NousResearch/terminal-bench*` on GitHub, the GitHub repositories API, raw content, and codeload. -The machine must provide `MAKA_EVAL_EGRESS_PROXY_URL` and a trusted CA file through -`MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH`. +benchmark and public-solution contamination URLs, including normalized or recursively wrapped +`terminal-bench` references, pinned benchmark revisions, task registries, benchmark repositories, +public trajectories, and known patch mirrors. The checked-in Compose overlay gives every cell its +own MITM proxy, CA, bounded audit log, and health gate. During `Agent.run()`, Harbor's Docker egress +sidecar applies an nftables allowlist containing only that proxy service; direct subject egress is +therefore rejected even when a command unsets proxy variables or requests `--noproxy`. Harbor task +download and verifier phases retain their native network policy. Build the pinned +`maka-eval-egress-proxy:12.2.3` image from `harbor/egress-proxy/Dockerfile` before running the +cohort. Collected Maka runtime files and egress audit logs are represented in attempt artifacts with +byte counts and SHA-256 digests. The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains. diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json index 7338980cea..83186871fe 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json @@ -30,16 +30,13 @@ "SSH_AUTH_SOCK" ], "egressProxy": { - "urlEnv": "MAKA_EVAL_EGRESS_PROXY_URL", - "caCertificatePathEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", + "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml", + "proxyUrl": "http://maka-eval-mitmproxy:8080", + "allowedHost": "maka-eval-mitmproxy", "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" }, "mounts": [ - { - "sourceEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", - "target": "/opt/maka-egress/mitmproxy-ca-cert.pem", - "readOnly": true - }, { "sourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "target": "/opt/maka-agent", diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json index a70f257d8f..e5895ab695 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json @@ -29,16 +29,13 @@ "no_proxy" ], "egressProxy": { - "urlEnv": "MAKA_EVAL_EGRESS_PROXY_URL", - "caCertificatePathEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", + "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", + "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml", + "proxyUrl": "http://maka-eval-mitmproxy:8080", + "allowedHost": "maka-eval-mitmproxy", "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" }, "mounts": [ - { - "sourceEnv": "MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH", - "target": "/opt/maka-egress/mitmproxy-ca-cert.pem", - "readOnly": true - }, { "sourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "target": "/opt/maka-agent", diff --git a/packages/eval/harbor/docker-compose-egress-proxy.yaml b/packages/eval/harbor/docker-compose-egress-proxy.yaml new file mode 100644 index 0000000000..d0ced39620 --- /dev/null +++ b/packages/eval/harbor/docker-compose-egress-proxy.yaml @@ -0,0 +1,26 @@ +services: + main: + depends_on: + maka-eval-mitmproxy: + condition: service_healthy + volumes: + - maka-eval-egress-state:/opt/maka-egress:ro + + maka-eval-mitmproxy: + image: maka-eval-egress-proxy:12.2.3 + networks: + - default + volumes: + - maka-eval-egress-state:/opt/maka-egress + healthcheck: + test: + - CMD + - python + - -c + - "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()" + interval: 1s + timeout: 3s + retries: 30 + +volumes: + maka-eval-egress-state: diff --git a/packages/eval/harbor/egress-proxy/Dockerfile b/packages/eval/harbor/egress-proxy/Dockerfile new file mode 100644 index 0000000000..550449f09f --- /dev/null +++ b/packages/eval/harbor/egress-proxy/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-slim + +RUN python -m pip install --no-cache-dir mitmproxy==12.2.3 + +COPY egress_filter.py /opt/maka-eval/egress_filter.py +COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh + +ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"] diff --git a/packages/eval/harbor/egress-proxy/entrypoint.sh b/packages/eval/harbor/egress-proxy/entrypoint.sh new file mode 100755 index 0000000000..1b81ea99df --- /dev/null +++ b/packages/eval/harbor/egress-proxy/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +mkdir -p /opt/maka-egress +: > /opt/maka-egress/hits.jsonl + +exec mitmdump \ + --quiet \ + --listen-host 0.0.0.0 \ + --listen-port 8080 \ + --set block_global=false \ + --set confdir=/opt/maka-egress \ + --scripts /opt/maka-eval/egress_filter.py diff --git a/packages/eval/harbor/egress_filter.py b/packages/eval/harbor/egress_filter.py index 941996b088..a429f211b4 100644 --- a/packages/eval/harbor/egress_filter.py +++ b/packages/eval/harbor/egress_filter.py @@ -1,32 +1,91 @@ -"""Path-level egress filter for Terminal-Bench evaluation subjects.""" +"""Fail-closed URL contamination filter for Eval subject egress.""" from __future__ import annotations +import json +import os +import re +import time +from pathlib import Path from urllib.parse import unquote, urlsplit -BLOCKED_OWNERS = {"harbor-framework", "nousresearch"} +PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932" +MAX_DECODE_PASSES = 4 +MAX_AUDIT_BYTES = 1024 * 1024 +AUDIT_PATH = Path(os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress/hits.jsonl")) +PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})") +TERMINAL_BENCH = re.compile(r"terminal[-_.%/]*bench", re.IGNORECASE) -def blocked_terminal_bench_url(raw_url: str) -> bool: - try: - url = urlsplit(raw_url) - except ValueError: - return False +def contamination_rule(raw_url: str) -> tuple[str, str, str] | None: + normalized = normalize_url(raw_url) + url = urlsplit(normalized) host = (url.hostname or "").lower().rstrip(".") - segments = [unquote(part).lower() for part in url.path.split("/") if part] - if host == "api.github.com": - return ( - len(segments) >= 3 - and segments[0] == "repos" - and blocked_repository(segments[1], segments[2]) - ) - if host in {"github.com", "raw.githubusercontent.com", "codeload.github.com"}: - return len(segments) >= 2 and blocked_repository(segments[0], segments[1]) - return False + path_query = f"{url.path}?{url.query}" if url.query else url.path + lowered = path_query.lower() + + if host == "r.jina.ai": + inner = unquote(url.path.lstrip("/")) + if inner.startswith(("http://", "https://")): + nested = contamination_rule(inner) + if nested: + return (f"jina_recursive:{nested[0]}", host, path_query) + + if PINNED_REVISION in lowered: + return ("pinned_revision", host, path_query) + if host == "tbench.ai": + return ("tbench_domain", host, path_query) + if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered: + return ("harbor_task_registry", host, path_query) + if benchmark_repository(host, lowered): + return ("benchmark_repository", host, path_query) + if public_trajectory_repository(host, lowered): + return ("public_trajectory", host, path_query) + if "patches-terminalbench-" in lowered: + return ("known_patch_artifact", host, path_query) + if TERMINAL_BENCH.search(lowered): + return ("terminal_bench_url", host, path_query) + return None + + +def normalize_url(raw_url: str) -> str: + value = raw_url.strip() + if not value: + raise ValueError("empty URL") + for _ in range(MAX_DECODE_PASSES): + if PERCENT_ESCAPE.search(value): + raise ValueError("malformed percent escape") + decoded = unquote(value) + if decoded == value: + break + value = decoded + else: + raise ValueError("URL exceeded decode limit") + parsed = urlsplit(value) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname: + raise ValueError("unsupported URL") + return value + + +def benchmark_repository(host: str, path_query: str) -> bool: + repositories = ( + "harbor-framework/terminal-bench", + "terminal-benchmarks/terminal-bench", + "tbench-ai/terminal-bench", + ) + return host in { + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + } and any(repository in path_query for repository in repositories) -def blocked_repository(owner: str, repository: str) -> bool: - return owner.lower() in BLOCKED_OWNERS and repository.lower().startswith("terminal-bench") +def public_trajectory_repository(host: str, path_query: str) -> bool: + return ( + host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"} + and "hqeric/maka-eval-trajectories" in path_query + ) try: @@ -38,11 +97,48 @@ def blocked_repository(owner: str, repository: str) -> bool: def request(flow: object) -> None: if http is None: raise RuntimeError("mitmproxy is required to run the Eval egress filter") - request_url = flow.request.pretty_url - if not blocked_terminal_bench_url(request_url): - return - flow.response = http.Response.make( + try: + matched = contamination_rule(flow.request.pretty_url) + if not matched: + return + rule_id, host, normalized_path = matched + append_audit(rule_id, host, normalized_path) + flow.response = blocked_response(rule_id) + except Exception as error: + flow.response = http.Response.make( + 503, + b"Eval egress policy could not classify this request.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": "policy_error", + }, + ) + try: + append_audit("policy_error", "", type(error).__name__) + except Exception: + pass + + +def blocked_response(rule_id: str): + return http.Response.make( 451, - b"Terminal-Bench benchmark source access is blocked during evaluation.\n", - {"Content-Type": "text/plain; charset=utf-8"}, + b"Benchmark source or public solution access is blocked during evaluation.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": rule_id, + }, ) + + +def append_audit(rule_id: str, host: str, normalized_path: str) -> None: + AUDIT_PATH.parent.mkdir(parents=True, exist_ok=True) + if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size >= MAX_AUDIT_BYTES: + return + record = { + "ts": int(time.time() * 1000), + "ruleId": rule_id, + "host": host[:255], + "normalizedPath": normalized_path[:4096], + } + with AUDIT_PATH.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n") diff --git a/packages/eval/harbor/run_trial.py b/packages/eval/harbor/run_trial.py index 146568025a..afbdbb3723 100644 --- a/packages/eval/harbor/run_trial.py +++ b/packages/eval/harbor/run_trial.py @@ -120,6 +120,7 @@ async def run_trial(framework: str, expected_version: str, config_file: Path) -> config.task.download_dir = namespace config.task.overwrite = ready is None trial = await trial_type.create(config) + apply_subject_egress_policy(trial) target = Path(trial.task.task_dir) if ready is not None and target.resolve() != ready: raise RuntimeError("Harbor task cache target changed for one identity") @@ -132,6 +133,15 @@ async def run_trial(framework: str, expected_version: str, config_file: Path) -> config_file.unlink(missing_ok=True) +def apply_subject_egress_policy(trial: object) -> None: + allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") + if not allowed_host: + return + agent = trial.task.config.agent + agent.network_mode = "allowlist" + agent.allowed_hosts = [allowed_host] + + async def main() -> None: framework, expected_version, config_path = sys.argv[1:] task = asyncio.current_task() diff --git a/packages/eval/harbor/test_egress_filter.py b/packages/eval/harbor/test_egress_filter.py index e85cfc1426..c9ba4a3bd7 100644 --- a/packages/eval/harbor/test_egress_filter.py +++ b/packages/eval/harbor/test_egress_filter.py @@ -1,6 +1,9 @@ import importlib.util +import json +import tempfile import unittest from pathlib import Path +from types import SimpleNamespace MODULE_PATH = Path(__file__).with_name("egress_filter.py") SPEC = importlib.util.spec_from_file_location("maka_eval_egress_filter", MODULE_PATH) @@ -10,25 +13,62 @@ class EgressFilterTest(unittest.TestCase): - def test_blocks_only_terminal_bench_repository_paths(self) -> None: - blocked = [ - "https://github.com/harbor-framework/terminal-bench-2", - "https://github.com/NousResearch/terminal-bench/tree/main", - "https://api.github.com/repos/harbor-framework/terminal-bench-2-1/contents/tasks", - "https://raw.githubusercontent.com/NousResearch/terminal-bench/main/README.md", - "https://codeload.github.com/harbor-framework/terminal-bench-2.1/tar.gz/main", - ] + def test_blocks_contamination_surfaces_and_recursive_jina_urls(self) -> None: + blocked = { + "https://github.com/harbor-framework/terminal-bench-2-1": "benchmark_repository", + "https://api.github.com/repos/terminal-benchmarks/terminal-bench/issues": "benchmark_repository", + "https://raw.githubusercontent.com/tbench-ai/terminal-bench/main/tests/x": "benchmark_repository", + "https://huggingface.co/datasets/acme/terminal-bench-traces": "terminal_bench_url", + "https://github.com/hqeric/maka-eval-trajectories": "public_trajectory", + "https://spylab.ai/reference/terminalbench-solution": "terminal_bench_url", + "https://example.test/patches-terminalbench-task-1.diff": "known_patch_artifact", + f"https://example.test/archive?revision={MODULE.PINNED_REVISION}": "pinned_revision", + "https://tbench.ai/tasks": "tbench_domain", + "https://hub.harborframework.com/tasks/terminal-bench/foo": "harbor_task_registry", + "https://r.jina.ai/https://github.com/harbor-framework/terminal-bench-2-1": "jina_recursive:benchmark_repository", + "https://r.jina.ai/https%253A%252F%252Fspylab.ai%252Fterminal-bench": "jina_recursive:terminal_bench_url", + "https://example.test/search?q=TeRmInAlBeNcH": "terminal_bench_url", + "https://github.com/harbor-framework/terminal%252Dbench-2-1.git": "benchmark_repository", + } + for url, rule_id in blocked.items(): + matched = MODULE.contamination_rule(url) + self.assertIsNotNone(matched, url) + self.assertEqual(matched[0], rule_id, url) + + def test_preserves_unrelated_network_and_rejects_malformed_urls(self) -> None: allowed = [ "https://github.com/harbor-framework/harbor", - "https://api.github.com/repos/other/terminal-bench", - "https://raw.githubusercontent.com/other/project/main/data.txt", + "https://github.com/microsoft/terminal", + "https://huggingface.co/datasets/mteb/leaderboard", "https://pypi.org/simple/requests/", - "https://huggingface.co/datasets/example/data", + "https://deb.debian.org/debian/", ] - for url in blocked: - self.assertTrue(MODULE.blocked_terminal_bench_url(url), url) for url in allowed: - self.assertFalse(MODULE.blocked_terminal_bench_url(url), url) + self.assertIsNone(MODULE.contamination_rule(url), url) + for url in ["", "file:///tmp/terminal-bench.log", "https://example.test/%ZZ"]: + with self.assertRaises(ValueError, msg=url): + MODULE.contamination_rule(url) + + def test_audit_is_bounded_and_policy_errors_fail_closed(self) -> None: + class Response: + @staticmethod + def make(status, body, headers): + return {"status": status, "body": body, "headers": headers} + + with tempfile.TemporaryDirectory() as directory: + MODULE.http = SimpleNamespace(Response=Response) + MODULE.AUDIT_PATH = Path(directory) / "hits.jsonl" + flow = type( + "Flow", + (), + {"request": type("Request", (), {"pretty_url": "https://example.test/%ZZ"})()}, + )() + MODULE.request(flow) + self.assertEqual(flow.response["status"], 503) + record = json.loads(MODULE.AUDIT_PATH.read_text().splitlines()[0]) + self.assertEqual(record["ruleId"], "policy_error") + self.assertIn("host", record) + self.assertIn("normalizedPath", record) if __name__ == "__main__": diff --git a/packages/eval/harbor/test_run_trial_policy.py b/packages/eval/harbor/test_run_trial_policy.py new file mode 100644 index 0000000000..0c49b4d20d --- /dev/null +++ b/packages/eval/harbor/test_run_trial_policy.py @@ -0,0 +1,30 @@ +import importlib.util +import os +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +MODULE_PATH = Path(__file__).with_name("run_trial.py") +SPEC = importlib.util.spec_from_file_location("maka_eval_run_trial", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class RunTrialPolicyTest(unittest.TestCase): + def test_forces_only_the_subject_phase_through_the_cell_proxy(self) -> None: + agent = SimpleNamespace(network_mode=None, allowed_hosts=None) + trial = SimpleNamespace(task=SimpleNamespace(config=SimpleNamespace(agent=agent))) + with patch.dict( + os.environ, + {"MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy"}, + clear=False, + ): + MODULE.apply_subject_egress_policy(trial) + self.assertEqual(agent.network_mode, "allowlist") + self.assertEqual(agent.allowed_hosts, ["maka-eval-mitmproxy"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/eval/package.json b/packages/eval/package.json index 3c9a52791d..8ed46892bc 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -19,10 +19,11 @@ "clean": "node ../../scripts/clean-paths.mjs dist tsconfig.tsbuildinfo", "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py && python3 harbor/test_egress_filter.py" + "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py && python3 harbor/test_egress_filter.py && python3 harbor/test_run_trial_policy.py" }, "dependencies": { "@maka/core": "0.1.0", - "@maka/runtime-host": "0.1.0" + "@maka/runtime-host": "0.1.0", + "undici": "^8.7.0" } } diff --git a/packages/eval/src/__tests__/external-subject.test.ts b/packages/eval/src/__tests__/external-subject.test.ts index c1610b362a..c15c861140 100644 --- a/packages/eval/src/__tests__/external-subject.test.ts +++ b/packages/eval/src/__tests__/external-subject.test.ts @@ -27,7 +27,7 @@ test('passes declared environment and credential bindings to one external comman metadata: {}, execute: async (input) => { request = input; - return { termination: 'exited', exitCode: 0, stdout: '' }; + return { termination: 'exited', exitCode: 0, stdout: '', stderr: '' }; }, }, }); @@ -195,6 +195,7 @@ test('verifies a mounted toolchain once before cell execution', async () => { costUsd: null, artifacts: [], }), + stderr: '', }; }, }, diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index 295b83bc54..aa9eaee1cc 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -449,7 +449,12 @@ test('Maka framework termination is authoritative before stdout decoding', async cwd: '/app', taskInput: 'solve', metadata: {}, - execute: async () => ({ termination: 'framework_timeout', exitCode: 124, stdout: '' }), + execute: async () => ({ + termination: 'framework_timeout', + exitCode: 124, + stdout: '', + stderr: '', + }), }, }); assert.equal(external.status, 'failed'); @@ -777,8 +782,10 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', config: { mounts: Array<{ target: string }>; egressProxy: { - urlEnv: string; - caCertificatePathEnv: string; + composeSourceEnv: string; + composeRelativePath: string; + proxyUrl: string; + allowedHost: string; containerCaPath: string; }; }; @@ -790,14 +797,23 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', ['maka', 'codex', 'claude-code', 'reasonix', 'opencode', 'kimi-code', 'zcode', 'pi'], ); assert.deepEqual(spec.executor.config.egressProxy, { - urlEnv: 'MAKA_EVAL_EGRESS_PROXY_URL', - caCertificatePathEnv: 'MAKA_EVAL_EGRESS_PROXY_CA_CERT_PATH', + composeSourceEnv: 'MAKA_EVAL_MAKA_BUNDLE_PATH', + composeRelativePath: 'packages/eval/harbor/docker-compose-egress-proxy.yaml', + proxyUrl: 'http://maka-eval-mitmproxy:8080', + allowedHost: 'maka-eval-mitmproxy', containerCaPath: '/opt/maka-egress/mitmproxy-ca-cert.pem', }); + const egressCompose = await readFile( + new URL('../../harbor/docker-compose-egress-proxy.yaml', import.meta.url), + 'utf8', + ); + assert.doesNotMatch(egressCompose, /^\s*ports:/mu); + assert.match(egressCompose, /condition: service_healthy/u); + assert.match(egressCompose, /maka-eval-egress-state:\/opt\/maka-egress/u); + assert.match(egressCompose, /networks:\s*\n\s+- default/u); assert.deepEqual( spec.executor.config.mounts.map(({ target }) => target), [ - '/opt/maka-egress/mitmproxy-ca-cert.pem', '/opt/maka-agent', '/opt/maka-node-toolchain', '/opt/maka-codex-toolchain', diff --git a/packages/eval/src/harbor-external-subject.ts b/packages/eval/src/harbor-external-subject.ts index 925f883b28..7b95fbbc15 100644 --- a/packages/eval/src/harbor-external-subject.ts +++ b/packages/eval/src/harbor-external-subject.ts @@ -5,6 +5,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { createServer, type IncomingMessage, type Server } from 'node:http'; import { basename, dirname, join } from 'node:path'; import type { Readable } from 'node:stream'; +import { fetch as undiciFetch, ProxyAgent } from 'undici'; import { decodePreverifiedToolchain, type ExternalProfile as Profile, @@ -600,6 +601,7 @@ async function startMeteringProxy( let admittedRequests = 0; let usageRequests = 0; let removedWebTools = 0; + const dispatcher = process.env.HTTPS_PROXY ? new ProxyAgent(process.env.HTTPS_PROXY) : undefined; const active = new Set>(); const server = createServer((request, response) => { const operation = (async () => { @@ -620,10 +622,11 @@ async function startMeteringProxy( } if (anthropic) headers.set('x-api-key', upstreamKey); else headers.set('authorization', `Bearer ${upstreamKey}`); - const upstream = await fetch(target, { + const upstream = await undiciFetch(target, { method: request.method, headers, body: projected.body.length === 0 ? undefined : new Uint8Array(projected.body), + ...(dispatcher ? { dispatcher } : {}), }); response.statusCode = upstream.status; upstream.headers.forEach((value, name) => { @@ -677,6 +680,7 @@ async function startMeteringProxy( close: async () => { await Promise.allSettled([...active]); await closeServer(server); + await dispatcher?.close(); }, }; } diff --git a/packages/eval/src/harbor-maka-subject.ts b/packages/eval/src/harbor-maka-subject.ts index 464c669fc5..447abb7e47 100644 --- a/packages/eval/src/harbor-maka-subject.ts +++ b/packages/eval/src/harbor-maka-subject.ts @@ -23,7 +23,7 @@ await mkdir(runtimeHome, { recursive: true, mode: 0o700 }); if (payload.webTools === 'disabled') { await writeFile( join(payload.rootPath, 'runtime-policy.json'), - `${JSON.stringify(disabledWebToolsRuntimePolicyDocument())}\n`, + `${JSON.stringify(disabledWebToolsRuntimePolicyDocument(process.env.HTTPS_PROXY))}\n`, { flag: 'wx', mode: 0o600 }, ); } diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index 1ee4b94401..7bf01cce64 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -1,5 +1,5 @@ import { spawn, type ChildProcess } from 'node:child_process'; -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { once } from 'node:events'; import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'; import { createServer, type Server, type Socket } from 'node:net'; @@ -345,7 +345,7 @@ async function startTrial( const trialPath = join(trialsRoot, trialName); const task = decodeTask(framework, options, cell); const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier'); - const environmentConfig = { ...options.environment, mounts: resolveMounts(options.mounts) }; + const environmentConfig = resolveEnvironmentConfig(options); const relayPath = resolve(dirname(fileURLToPath(import.meta.url)), '../harbor'); const executionEnvironment = egressExecutionEnvironment(options.egressProxy); const environment = preparationEnvironment( @@ -353,6 +353,7 @@ async function startTrial( relayPath, [...subjectCredentialNames, ...cell.subject.credentials], options.preparationEnvironment, + options.egressProxy?.allowedHost, ); const server = createServer(); const connections = new Set(); @@ -393,6 +394,17 @@ async function startTrial( }, }, environment: environmentConfig, + ...(options.egressProxy + ? { + artifacts: [ + { + source: '/opt/maka-egress/hits.jsonl', + destination: 'egress-hits.jsonl', + service: 'maka-eval-mitmproxy', + }, + ], + } + : {}), })}\n`, { flag: 'wx', mode: 0o600 }, ); @@ -572,6 +584,7 @@ function preparationEnvironment( relayPath: string, subjectCredentialNames: readonly string[], declared: readonly string[], + egressAllowedHost?: string, ): NodeJS.ProcessEnv { const allowed = new Set([ 'HOME', @@ -599,6 +612,7 @@ function preparationEnvironment( ...inherited, MAKA_EVAL_FRAMEWORK: framework, PYTHONPATH: [relayPath, inherited.PYTHONPATH].filter(Boolean).join(delimiter), + ...(egressAllowedHost ? { MAKA_EVAL_EGRESS_ALLOWED_HOST: egressAllowedHost } : {}), }; } @@ -617,17 +631,12 @@ function egressExecutionEnvironment( options: HarnessOptions['egressProxy'], ): Readonly> { if (!options) return {}; - const proxyUrl = process.env[options.urlEnv]!; - if (!URL.canParse(proxyUrl) || new URL(proxyUrl).protocol !== 'http:') { - throw new Error(`machine path ${options.urlEnv} must be an HTTP proxy URL`); - } - const proxyHost = new URL(proxyUrl).hostname; - const noProxy = `127.0.0.1,localhost,${proxyHost}`; + const noProxy = '127.0.0.1,localhost'; return { - HTTP_PROXY: proxyUrl, - HTTPS_PROXY: proxyUrl, - http_proxy: proxyUrl, - https_proxy: proxyUrl, + HTTP_PROXY: options.proxyUrl, + HTTPS_PROXY: options.proxyUrl, + http_proxy: options.proxyUrl, + https_proxy: options.proxyUrl, NO_PROXY: noProxy, no_proxy: noProxy, SSL_CERT_FILE: options.containerCaPath, @@ -665,11 +674,25 @@ async function readVerification( if (result.exception_info && !subjectException) { throw new Error('Trial failed outside subject execution'); } + const egressAuditPath = join(state.trialPath, 'artifacts', 'egress-hits.jsonl'); + const egressAudit = await readFile(egressAuditPath).catch(() => undefined); return { status: score === null ? 'infra_failed' : subjectException ? 'subject_failed' : 'completed', score, failureReason: score === null ? 'verifier produced no reward' : null, - artifacts: [{ kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }], + artifacts: [ + { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, + ...(egressAudit + ? [ + { + kind: 'egress-audit', + path: 'artifacts/egress-hits.jsonl', + bytes: egressAudit.byteLength, + sha256: createHash('sha256').update(egressAudit).digest('hex'), + }, + ] + : []), + ], }; } @@ -681,8 +704,10 @@ interface HarnessOptions { readonly environment: JsonObject; readonly preparationEnvironment: readonly string[]; readonly egressProxy?: { - readonly urlEnv: string; - readonly caCertificatePathEnv: string; + readonly composeSourceEnv: string; + readonly composeRelativePath: string; + readonly proxyUrl: string; + readonly allowedHost: string; readonly containerCaPath: string; }; readonly mounts: readonly { @@ -735,8 +760,7 @@ function decodeOptions(value: JsonObject, framework: Framework): HarnessOptions decoded.pythonPathEnv, decoded.trialsRootEnv, decoded.tasksRootEnv, - decoded.egressProxy?.urlEnv, - decoded.egressProxy?.caCertificatePathEnv, + decoded.egressProxy?.composeSourceEnv, ]) { if (name && !process.env[name]) throw new Error(`machine path ${name} is unavailable`); } @@ -744,13 +768,20 @@ function decodeOptions(value: JsonObject, framework: Framework): HarnessOptions } function decodeEgressProxy(value: unknown): NonNullable { - const proxy = exact(value, ['urlEnv', 'caCertificatePathEnv', 'containerCaPath'], 'egressProxy'); + const proxy = exact( + value, + ['composeSourceEnv', 'composeRelativePath', 'proxyUrl', 'allowedHost', 'containerCaPath'], + 'egressProxy', + ); + const proxyUrl = text(proxy.proxyUrl, 'egressProxy.proxyUrl'); + if (!URL.canParse(proxyUrl) || new URL(proxyUrl).protocol !== 'http:') { + throw new Error('egressProxy.proxyUrl must be an HTTP proxy URL'); + } return { - urlEnv: machinePathEnv(proxy.urlEnv, 'egressProxy.urlEnv'), - caCertificatePathEnv: machinePathEnv( - proxy.caCertificatePathEnv, - 'egressProxy.caCertificatePathEnv', - ), + composeSourceEnv: machinePathEnv(proxy.composeSourceEnv, 'egressProxy.composeSourceEnv'), + composeRelativePath: relativePath(proxy.composeRelativePath, 'egressProxy.composeRelativePath'), + proxyUrl, + allowedHost: hostName(proxy.allowedHost, 'egressProxy.allowedHost'), containerCaPath: absolute(proxy.containerCaPath, 'egressProxy.containerCaPath'), }; } @@ -776,6 +807,17 @@ function resolveMounts(mounts: HarnessOptions['mounts']) { })); } +function resolveEnvironmentConfig(options: HarnessOptions): JsonObject { + const base = { ...options.environment, mounts: resolveMounts(options.mounts) }; + if (!options.egressProxy) return base; + const source = resolve(process.env[options.egressProxy.composeSourceEnv]!); + const composePath = resolve(source, options.egressProxy.composeRelativePath); + if (relative(source, composePath).startsWith(`..${sep}`)) { + throw new Error('egress proxy compose path escapes its source root'); + } + return { ...base, extra_docker_compose: [composePath] }; +} + function decodeTask(framework: Framework, options: HarnessOptions, cell: ExperimentCell) { if (framework === 'harbor') { const benchmark = exact(cell.benchmark.config, ['repository'], 'benchmark.config'); @@ -972,6 +1014,22 @@ function machinePathEnv(value: unknown, where: string): string { return name; } +function relativePath(value: unknown, where: string): string { + const path = text(value, where); + if (path.startsWith('/') || path === '..' || path.startsWith(`..${sep}`)) { + throw new Error(`${where} must stay within its source root`); + } + return path; +} + +function hostName(value: unknown, where: string): string { + const host = text(value, where).toLowerCase(); + if (!/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/u.test(host)) { + throw new Error(`${where} must be a hostname`); + } + return host; +} + function absolute(value: unknown, where: string): string { const path = text(value, where); if (!path.startsWith('/')) throw new Error(`${where} must be absolute`); diff --git a/packages/eval/src/maka-runtime-policy.ts b/packages/eval/src/maka-runtime-policy.ts index 7591fd490f..2ae30abd28 100644 --- a/packages/eval/src/maka-runtime-policy.ts +++ b/packages/eval/src/maka-runtime-policy.ts @@ -1,12 +1,24 @@ import { createDefaultRuntimePolicy } from '@maka/core/runtime-policy'; -export function disabledWebToolsRuntimePolicyDocument() { +export function disabledWebToolsRuntimePolicyDocument(proxyUrl?: string) { const policy = createDefaultRuntimePolicy(); + const proxy = proxyUrl ? new URL(proxyUrl) : undefined; return { schemaVersion: 2 as const, revision: 0, policy: { ...policy, + ...(proxy + ? { + networkProxy: { + ...policy.networkProxy, + enabled: true, + protocol: 'http' as const, + host: proxy.hostname, + port: Number(proxy.port || 80), + }, + } + : {}), privacy: { incognitoActive: true }, webSearch: { ...policy.webSearch, enabled: false }, }, From 6b242f9d131630a3526e661fa1f0ac917fe3f39c Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:29:16 +0000 Subject: [PATCH 07/13] fix(eval): apply egress policy before Harbor environment creation --- packages/eval/harbor/run_trial.py | 28 ++++++++++++++++--- packages/eval/harbor/test_run_trial_policy.py | 4 +-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/eval/harbor/run_trial.py b/packages/eval/harbor/run_trial.py index afbdbb3723..29eb39e021 100644 --- a/packages/eval/harbor/run_trial.py +++ b/packages/eval/harbor/run_trial.py @@ -119,8 +119,7 @@ async def run_trial(framework: str, expected_version: str, config_file: Path) -> namespace.mkdir(parents=True, exist_ok=True, mode=0o700) config.task.download_dir = namespace config.task.overwrite = ready is None - trial = await trial_type.create(config) - apply_subject_egress_policy(trial) + trial = await create_harbor_trial(trial_type, config) target = Path(trial.task.task_dir) if ready is not None and target.resolve() != ready: raise RuntimeError("Harbor task cache target changed for one identity") @@ -133,15 +132,36 @@ async def run_trial(framework: str, expected_version: str, config_file: Path) -> config_file.unlink(missing_ok=True) -def apply_subject_egress_policy(trial: object) -> None: +def apply_subject_egress_policy(task: object) -> None: allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") if not allowed_host: return - agent = trial.task.config.agent + agent = task.config.agent agent.network_mode = "allowlist" agent.allowed_hosts = [allowed_host] +async def create_harbor_trial(trial_type: type, config: object) -> object: + trial_type._resolve_agent_skills(config) + task, task_download_result = await trial_type._load_task(config) + apply_subject_egress_policy(task) + if task.has_steps: + from harbor.trial.multi_step import MultiStepTrial + + return MultiStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + from harbor.trial.single_step import SingleStepTrial + + return SingleStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + + async def main() -> None: framework, expected_version, config_path = sys.argv[1:] task = asyncio.current_task() diff --git a/packages/eval/harbor/test_run_trial_policy.py b/packages/eval/harbor/test_run_trial_policy.py index 0c49b4d20d..4f251541a6 100644 --- a/packages/eval/harbor/test_run_trial_policy.py +++ b/packages/eval/harbor/test_run_trial_policy.py @@ -15,13 +15,13 @@ class RunTrialPolicyTest(unittest.TestCase): def test_forces_only_the_subject_phase_through_the_cell_proxy(self) -> None: agent = SimpleNamespace(network_mode=None, allowed_hosts=None) - trial = SimpleNamespace(task=SimpleNamespace(config=SimpleNamespace(agent=agent))) + task = SimpleNamespace(config=SimpleNamespace(agent=agent)) with patch.dict( os.environ, {"MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy"}, clear=False, ): - MODULE.apply_subject_egress_policy(trial) + MODULE.apply_subject_egress_policy(task) self.assertEqual(agent.network_mode, "allowlist") self.assertEqual(agent.allowed_hosts, ["maka-eval-mitmproxy"]) From 4fdd8535ecf70a4bb0c4460f2cdc49d830175f6a Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:48:24 +0000 Subject: [PATCH 08/13] fix(eval): restrict subject namespace to the cell proxy --- .../harbor/docker-compose-egress-proxy.yaml | 7 ++ .../eval/harbor/egress-proxy/network-policy | 90 +++++++++++++++++++ .../__tests__/lifecycle-boundaries.test.ts | 1 + 3 files changed, 98 insertions(+) create mode 100755 packages/eval/harbor/egress-proxy/network-policy diff --git a/packages/eval/harbor/docker-compose-egress-proxy.yaml b/packages/eval/harbor/docker-compose-egress-proxy.yaml index d0ced39620..de7520aa15 100644 --- a/packages/eval/harbor/docker-compose-egress-proxy.yaml +++ b/packages/eval/harbor/docker-compose-egress-proxy.yaml @@ -1,4 +1,11 @@ services: + harbor-docker-egress-control-sidecar: + volumes: + - type: bind + source: ./egress-proxy/network-policy + target: /usr/local/bin/network-policy + read_only: true + main: depends_on: maka-eval-mitmproxy: diff --git a/packages/eval/harbor/egress-proxy/network-policy b/packages/eval/harbor/egress-proxy/network-policy new file mode 100755 index 0000000000..638cbb48d1 --- /dev/null +++ b/packages/eval/harbor/egress-proxy/network-policy @@ -0,0 +1,90 @@ +#!/bin/sh +set -eu + +ALLOWLIST=/opt/egress-sidecar/allowlist.txt +GOST_PORT=12345 +GOST_MARK=114514 +NFTABLES_RULESET_NAME=gost_egress +PROXY_HOST=maka-eval-mitmproxy +PROXY_PORT=8080 + +remove_nftables() { + nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true +} + +resolve_proxy_ip() { + getent ahostsv4 "$PROXY_HOST" | awk 'NR == 1 { print $1 }' +} + +setup_proxy_only() { + proxy_ip="$(resolve_proxy_ip)" + if [ -z "$proxy_ip" ]; then + echo "could not resolve Eval egress proxy" >&2 + exit 1 + fi + : > "$ALLOWLIST" + sleep 2 + nft --file - < "$ALLOWLIST" + echo "ok: allow all egress" + ;; + deny-all) + : > "$ALLOWLIST" + sleep 2 + setup_proxy_only + ;; + allow) + shift + if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then + echo "Eval egress policy accepts only $PROXY_HOST" >&2 + exit 2 + fi + setup_proxy_only + ;; + show) + if nft list table inet "$NFTABLES_RULESET_NAME" >/dev/null 2>&1; then + echo "mode: Eval proxy-only" + echo "proxy: $PROXY_HOST:$PROXY_PORT" + else + echo "mode: allow-all" + fi + ;; + rules) + nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all" + ;; + *) + echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2 + exit 2 + ;; +esac diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index aa9eaee1cc..e8251ff18b 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -811,6 +811,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', assert.match(egressCompose, /condition: service_healthy/u); assert.match(egressCompose, /maka-eval-egress-state:\/opt\/maka-egress/u); assert.match(egressCompose, /networks:\s*\n\s+- default/u); + assert.match(egressCompose, /target: \/usr\/local\/bin\/network-policy/u); assert.deepEqual( spec.executor.config.mounts.map(({ target }) => target), [ From 1329c069f630c6ffa971a0fcd823693c41d40887 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 07:52:03 +0000 Subject: [PATCH 09/13] fix(eval): resolve the network policy mount from the bundle --- ...bench-2.1-deepseek-v4-flash-eight-arm.json | 1 + ...-bench-2.1-deepseek-v4-flash-four-arm.json | 1 + .../harbor/docker-compose-egress-proxy.yaml | 2 +- .../__tests__/lifecycle-boundaries.test.ts | 2 ++ packages/eval/src/harness-executor.ts | 28 ++++++++++++++++++- 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json index 83186871fe..e8f8bd9d0c 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json @@ -32,6 +32,7 @@ "egressProxy": { "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml", + "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy", "proxyUrl": "http://maka-eval-mitmproxy:8080", "allowedHost": "maka-eval-mitmproxy", "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json index e5895ab695..72cc1fc86c 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json @@ -31,6 +31,7 @@ "egressProxy": { "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH", "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml", + "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy", "proxyUrl": "http://maka-eval-mitmproxy:8080", "allowedHost": "maka-eval-mitmproxy", "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem" diff --git a/packages/eval/harbor/docker-compose-egress-proxy.yaml b/packages/eval/harbor/docker-compose-egress-proxy.yaml index de7520aa15..156281ed62 100644 --- a/packages/eval/harbor/docker-compose-egress-proxy.yaml +++ b/packages/eval/harbor/docker-compose-egress-proxy.yaml @@ -2,7 +2,7 @@ services: harbor-docker-egress-control-sidecar: volumes: - type: bind - source: ./egress-proxy/network-policy + source: ${MAKA_EVAL_NETWORK_POLICY_PATH} target: /usr/local/bin/network-policy read_only: true diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index e8251ff18b..701ed33230 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -784,6 +784,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', egressProxy: { composeSourceEnv: string; composeRelativePath: string; + networkPolicyRelativePath: string; proxyUrl: string; allowedHost: string; containerCaPath: string; @@ -799,6 +800,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', assert.deepEqual(spec.executor.config.egressProxy, { composeSourceEnv: 'MAKA_EVAL_MAKA_BUNDLE_PATH', composeRelativePath: 'packages/eval/harbor/docker-compose-egress-proxy.yaml', + networkPolicyRelativePath: 'packages/eval/harbor/egress-proxy/network-policy', proxyUrl: 'http://maka-eval-mitmproxy:8080', allowedHost: 'maka-eval-mitmproxy', containerCaPath: '/opt/maka-egress/mitmproxy-ca-cert.pem', diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index 7bf01cce64..fd9fb3fb41 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -346,6 +346,7 @@ async function startTrial( const task = decodeTask(framework, options, cell); const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier'); const environmentConfig = resolveEnvironmentConfig(options); + const networkPolicyPath = resolveNetworkPolicyPath(options); const relayPath = resolve(dirname(fileURLToPath(import.meta.url)), '../harbor'); const executionEnvironment = egressExecutionEnvironment(options.egressProxy); const environment = preparationEnvironment( @@ -354,6 +355,7 @@ async function startTrial( [...subjectCredentialNames, ...cell.subject.credentials], options.preparationEnvironment, options.egressProxy?.allowedHost, + networkPolicyPath, ); const server = createServer(); const connections = new Set(); @@ -585,6 +587,7 @@ function preparationEnvironment( subjectCredentialNames: readonly string[], declared: readonly string[], egressAllowedHost?: string, + networkPolicyPath?: string, ): NodeJS.ProcessEnv { const allowed = new Set([ 'HOME', @@ -613,6 +616,7 @@ function preparationEnvironment( MAKA_EVAL_FRAMEWORK: framework, PYTHONPATH: [relayPath, inherited.PYTHONPATH].filter(Boolean).join(delimiter), ...(egressAllowedHost ? { MAKA_EVAL_EGRESS_ALLOWED_HOST: egressAllowedHost } : {}), + ...(networkPolicyPath ? { MAKA_EVAL_NETWORK_POLICY_PATH: networkPolicyPath } : {}), }; } @@ -706,6 +710,7 @@ interface HarnessOptions { readonly egressProxy?: { readonly composeSourceEnv: string; readonly composeRelativePath: string; + readonly networkPolicyRelativePath: string; readonly proxyUrl: string; readonly allowedHost: string; readonly containerCaPath: string; @@ -770,7 +775,14 @@ function decodeOptions(value: JsonObject, framework: Framework): HarnessOptions function decodeEgressProxy(value: unknown): NonNullable { const proxy = exact( value, - ['composeSourceEnv', 'composeRelativePath', 'proxyUrl', 'allowedHost', 'containerCaPath'], + [ + 'composeSourceEnv', + 'composeRelativePath', + 'networkPolicyRelativePath', + 'proxyUrl', + 'allowedHost', + 'containerCaPath', + ], 'egressProxy', ); const proxyUrl = text(proxy.proxyUrl, 'egressProxy.proxyUrl'); @@ -780,6 +792,10 @@ function decodeEgressProxy(value: unknown): NonNullable Date: Thu, 13 Aug 2026 08:21:12 +0000 Subject: [PATCH 10/13] fix(eval): expose filtered provider request diagnostics --- .../__tests__/lifecycle-boundaries.test.ts | 7 ++++- .../provider-web-tool-surface.test.ts | 2 ++ packages/eval/src/harbor-external-subject.ts | 19 ++++++++++-- .../eval/src/provider-web-tool-surface.ts | 30 +++++++++++++++---- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index 701ed33230..d2f3dd23bb 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -705,6 +705,11 @@ test('eight-arm spec and wrappers freeze the working provider contracts', async await readFile(join(root, 'etc/claude-code/managed-settings.json'), 'utf8'), /WebSearch.*WebFetch/u, ); + assert.equal((await stat(join(root, 'etc/claude-code'))).mode & 0o777, 0o755); + assert.equal( + (await stat(join(root, 'etc/claude-code/managed-settings.json'))).mode & 0o777, + 0o644, + ); assert.match( await readFile(join(root, 'tmp/maka-eval-reasonix/config.toml'), 'utf8'), /enabled = \["bash", "read_file", "write_file", "edit_file", "glob", "grep"\]/u, diff --git a/packages/eval/src/__tests__/provider-web-tool-surface.test.ts b/packages/eval/src/__tests__/provider-web-tool-surface.test.ts index 92c72f8187..2053044045 100644 --- a/packages/eval/src/__tests__/provider-web-tool-surface.test.ts +++ b/packages/eval/src/__tests__/provider-web-tool-surface.test.ts @@ -20,6 +20,8 @@ test('Eval removes named and provider-native web tools from provider requests', ); assert.equal(projected.removed, 5); + assert.equal(projected.model, 'deepseek-v4-flash'); + assert.deepEqual(projected.toolNames, ['Read']); assert.deepEqual(JSON.parse(projected.body.toString('utf8')), { model: 'deepseek-v4-flash', tools: [{ type: 'function', function: { name: 'Read' } }], diff --git a/packages/eval/src/harbor-external-subject.ts b/packages/eval/src/harbor-external-subject.ts index 7b95fbbc15..372bf60f81 100644 --- a/packages/eval/src/harbor-external-subject.ts +++ b/packages/eval/src/harbor-external-subject.ts @@ -1,7 +1,7 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import { createHash } from 'node:crypto'; import { readFileSync, rmSync, statSync } from 'node:fs'; -import { mkdir, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, writeFile } from 'node:fs/promises'; import { createServer, type IncomingMessage, type Server } from 'node:http'; import { basename, dirname, join } from 'node:path'; import type { Readable } from 'node:stream'; @@ -107,6 +107,8 @@ try { usageRequests: proxy.usageRequestCount(), usageComplete: proxy.usageComplete(), removedWebTools: proxy.removedWebToolCount(), + models: proxy.requestModels(), + toolNames: proxy.observedToolNames(), }, fileArtifact('wrapper-state', statePath, profile), ); @@ -593,6 +595,8 @@ async function startMeteringProxy( usageRequestCount(): number; usageComplete(): boolean; removedWebToolCount(): number; + requestModels(): readonly string[]; + observedToolNames(): readonly string[]; close(): Promise; }> { const total = zeroUsage(); @@ -601,6 +605,8 @@ async function startMeteringProxy( let admittedRequests = 0; let usageRequests = 0; let removedWebTools = 0; + const requestModels = new Set(); + const observedToolNames = new Set(); const dispatcher = process.env.HTTPS_PROXY ? new ProxyAgent(process.env.HTTPS_PROXY) : undefined; const active = new Set>(); const server = createServer((request, response) => { @@ -608,6 +614,8 @@ async function startMeteringProxy( requests += 1; const projected = removeEvalWebTools(await readRequest(request)); removedWebTools += projected.removed; + if (projected.model) requestModels.add(projected.model); + for (const name of projected.toolNames) observedToolNames.add(name); const target = joinUpstream(upstreamBaseUrl, request.url ?? '/'); const headers = new Headers(); for (const [name, value] of Object.entries(request.headers)) { @@ -677,6 +685,8 @@ async function startMeteringProxy( usageRequestCount: () => usageRequests, usageComplete: () => admittedRequests > 0 && usageRequests === admittedRequests, removedWebToolCount: () => removedWebTools, + requestModels: () => [...requestModels].sort(), + observedToolNames: () => [...observedToolNames].sort(), close: async () => { await Promise.allSettled([...active]); await closeServer(server); @@ -838,8 +848,11 @@ function fileArtifact(kind: string, path: string, selected: Profile): Record { - await mkdir(directory, { recursive: true }); - await writeFile(join(directory, name), contents); + await mkdir(directory, { recursive: true, mode: 0o755 }); + await chmod(directory, 0o755); + const path = join(directory, name); + await writeFile(path, contents, { mode: 0o644 }); + await chmod(path, 0o644); } function rooted(root: string, absolutePath: string): string { diff --git a/packages/eval/src/provider-web-tool-surface.ts b/packages/eval/src/provider-web-tool-surface.ts index bbc11ce7be..41b25bbb11 100644 --- a/packages/eval/src/provider-web-tool-surface.ts +++ b/packages/eval/src/provider-web-tool-surface.ts @@ -3,19 +3,30 @@ const DISALLOWED_WEB_TOOL_NAMES = new Set(['websearch', 'webfetch', 'fetchurl']) export function removeEvalWebTools(body: Buffer): { readonly body: Buffer; readonly removed: number; + readonly model?: string; + readonly toolNames: readonly string[]; } { - if (body.length === 0) return { body, removed: 0 }; + if (body.length === 0) return { body, removed: 0, toolNames: [] }; let value: unknown; try { value = JSON.parse(body.toString('utf8')); } catch { - return { body, removed: 0 }; + return { body, removed: 0, toolNames: [] }; } - if (!isRecord(value) || !Array.isArray(value.tools)) return { body, removed: 0 }; + if (!isRecord(value)) return { body, removed: 0, toolNames: [] }; + const model = typeof value.model === 'string' ? value.model : undefined; + if (!Array.isArray(value.tools)) + return { body, removed: 0, toolNames: [], ...(model ? { model } : {}) }; const tools = value.tools.filter((tool) => !isDisallowedWebTool(tool)); const removed = value.tools.length - tools.length; - if (removed === 0) return { body, removed: 0 }; - return { body: Buffer.from(JSON.stringify({ ...value, tools }), 'utf8'), removed }; + const toolNames = tools.flatMap(toolName); + if (removed === 0) return { body, removed: 0, toolNames, ...(model ? { model } : {}) }; + return { + body: Buffer.from(JSON.stringify({ ...value, tools }), 'utf8'), + removed, + toolNames, + ...(model ? { model } : {}), + }; } function isDisallowedWebTool(tool: unknown): boolean { @@ -35,6 +46,15 @@ function normalize(value: string): string { return value.toLowerCase().replace(/[^a-z0-9]/gu, ''); } +function toolName(tool: unknown): string[] { + if (!isRecord(tool)) return []; + if (typeof tool.name === 'string') return [tool.name]; + if (isRecord(tool.function) && typeof tool.function.name === 'string') { + return [tool.function.name]; + } + return typeof tool.type === 'string' ? [tool.type] : []; +} + function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } From 2df0766c5da8e7ed01a1f9cd4abb125436933d5c Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 08:26:43 +0000 Subject: [PATCH 11/13] fix(eval): use portable undici request headers --- packages/eval/src/harbor-external-subject.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/eval/src/harbor-external-subject.ts b/packages/eval/src/harbor-external-subject.ts index 372bf60f81..afadc90160 100644 --- a/packages/eval/src/harbor-external-subject.ts +++ b/packages/eval/src/harbor-external-subject.ts @@ -617,7 +617,7 @@ async function startMeteringProxy( if (projected.model) requestModels.add(projected.model); for (const name of projected.toolNames) observedToolNames.add(name); const target = joinUpstream(upstreamBaseUrl, request.url ?? '/'); - const headers = new Headers(); + const headers: Record = {}; for (const [name, value] of Object.entries(request.headers)) { if ( value === undefined || @@ -626,10 +626,10 @@ async function startMeteringProxy( ) ) continue; - headers.set(name, Array.isArray(value) ? value.join(', ') : value); + headers[name] = Array.isArray(value) ? value.join(', ') : value; } - if (anthropic) headers.set('x-api-key', upstreamKey); - else headers.set('authorization', `Bearer ${upstreamKey}`); + if (anthropic) headers['x-api-key'] = upstreamKey; + else headers.authorization = `Bearer ${upstreamKey}`; const upstream = await undiciFetch(target, { method: request.method, headers, From 11a947037d1ca2aa80d80baa5cf61c4af4e9ddd9 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 09:46:09 +0000 Subject: [PATCH 12/13] fix(eval): freeze the durable hosted execution contract --- packages/core/src/runtime-inputs.ts | 3 + packages/core/src/session.ts | 9 + packages/eval/README.md | 24 +- ...bench-2.1-deepseek-v4-flash-eight-arm.json | 1 - ...-bench-2.1-deepseek-v4-flash-four-arm.json | 1 - packages/eval/harbor/egress_filter.py | 2 +- packages/eval/harbor/relay_agent.py | 35 ++- packages/eval/harbor/run_trial.py | 5 +- packages/eval/harbor/test_egress_filter.py | 1 + packages/eval/harbor/test_relay_artifacts.py | 66 +++++ packages/eval/harbor/test_relay_lifecycle.py | 8 + packages/eval/harbor/test_run_trial_policy.py | 11 +- packages/eval/package.json | 2 +- .../__tests__/lifecycle-boundaries.test.ts | 11 +- .../eval/src/__tests__/maka-artifacts.test.ts | 147 ++++++++++ packages/eval/src/harbor-maka-subject.ts | 61 ++-- packages/eval/src/harness-executor.ts | 60 +++- packages/eval/src/maka-artifacts.ts | 135 +++++++++ packages/eval/src/maka-runtime-policy.ts | 3 +- packages/eval/src/maka-subject.ts | 52 +++- .../execution-model-composition.test.ts | 272 +++++++++++++++++- .../hosted-execution-tool-profile.test.ts | 77 +++-- .../src/protocol/hosted-execution.ts | 19 +- .../src/protocol/session-catalog.ts | 12 + .../src/server/execution-composition.ts | 24 +- .../server/hosted-execution-tool-profile.ts | 82 +++--- .../src/server/interactive-run-composer.ts | 27 +- .../src/server/session-catalog-coordinator.ts | 4 +- .../sqlite-session-metadata-store.test.ts | 1 + packages/storage/src/session-store.ts | 17 +- 30 files changed, 1004 insertions(+), 168 deletions(-) create mode 100644 packages/eval/harbor/test_relay_artifacts.py create mode 100644 packages/eval/src/__tests__/maka-artifacts.test.ts create mode 100644 packages/eval/src/maka-artifacts.ts diff --git a/packages/core/src/runtime-inputs.ts b/packages/core/src/runtime-inputs.ts index 47ab065f0b..994166557f 100644 --- a/packages/core/src/runtime-inputs.ts +++ b/packages/core/src/runtime-inputs.ts @@ -7,6 +7,7 @@ import type { BackendKind, SessionBlockedReason, SessionStatus, + SessionToolProfile, SubagentSessionParent, SubagentSessionRuntime, SubagentSessionSpawn, @@ -37,6 +38,8 @@ export interface CreateSessionInput { model?: string; /** Per-model reasoning-depth variant; `undefined` = model default. */ thinkingLevel?: ThinkingLevel; + /** Immutable versioned prompt/tool contract for this Session. */ + toolProfile?: SessionToolProfile; permissionMode: PermissionMode; /** Defaults to `agent`. */ collaborationMode?: CollaborationMode; diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index b464bac679..0d6217c144 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -174,6 +174,13 @@ export function isTurnStatus(value: unknown): value is TurnStatus { // Header (JSONL line 1) // ============================================================================ +export const SESSION_TOOL_PROFILES = ['headless-coding-v1'] as const; +export type SessionToolProfile = (typeof SESSION_TOOL_PROFILES)[number]; + +export function isSessionToolProfile(value: unknown): value is SessionToolProfile { + return typeof value === 'string' && (SESSION_TOOL_PROFILES as readonly string[]).includes(value); +} + export interface SessionHeader { // Identity id: string; @@ -233,6 +240,8 @@ export interface SessionHeader { connectionLocked: boolean; /** Sticky session default model id, captured when the session is created. */ model: string; + /** Immutable versioned prompt/tool contract for non-product execution surfaces. */ + toolProfile?: SessionToolProfile; /** Per-model reasoning-depth variant; `undefined` = model default. Cleared on model switch. */ thinkingLevel?: import('./model-thinking.js').ThinkingLevel; permissionMode: PermissionMode; diff --git a/packages/eval/README.md b/packages/eval/README.md index 0d75bd256c..a00822c60f 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -26,13 +26,15 @@ The result kernel contains only score, normalized usage, attributable cost, dura The checked-in Terminal-Bench 2.1 four-arm cohort is `experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json`. It freezes provider endpoints, framework version, container paths and read-only mount policy. Set each declared machine-path environment variable to its trusted prepared directory, and set the declared API-key credentials. Machine-local paths select artifacts; they do not alter experiment semantics and are not presented as a cryptographic identity scheme. -Maka benchmark subjects also freeze a versioned Hosted Execution tool profile. The -`headless-coding-v1` profile admits only `Bash`, `Read`, `Write`, `Edit`, `Glob`, `Grep`, and -`apply_patch` as agent-permission tool candidates. Provider-specific routing remains authoritative: -DeepSeek Responses exposes `apply_patch` instead of `Write` and `Edit`. Runtime-owned `ArchiveRead` -remains available when context budget pruning creates an archive placeholder. Long-term-memory -triggers and product task, goal, scheduling, skill, interaction, background-control, and parent-agent -tools are outside this profile and must not drift into a benchmark run. +Maka benchmark subjects freeze a versioned Session profile. `headless-coding-v1` is persisted in +the Session header, so later turns and backend rebuilds retain the same contract. It fixes the +system prompt, disables product identity/personalization/skills/workspace-memory prompt fragments, +admits only `Bash`, `Read`, `Write`, `Edit`, `Glob`, `Grep`, and `apply_patch` as tool candidates, +and exposes a foreground-only Bash schema without `run_in_background` or `pty`. Provider-specific +routing remains authoritative: DeepSeek Responses exposes `apply_patch` instead of `Write` and +`Edit`, and Runtime-owned `ArchiveRead` remains available for archived tool results. A real +`hosted.execution.start` regression test pins SHA-256 hashes for the first main provider request's +developer prompt and complete tool schema. Every benchmark subject removes `WebSearch`, `WebFetch`, and `FetchURL` from the provider-visible tool list. Maka enforces that through its Hosted Execution profile; external harnesses pass through @@ -46,7 +48,11 @@ sidecar applies an nftables allowlist containing only that proxy service; direct therefore rejected even when a command unsets proxy variables or requests `--noproxy`. Harbor task download and verifier phases retain their native network policy. Build the pinned `maka-eval-egress-proxy:12.2.3` image from `harbor/egress-proxy/Dockerfile` before running the -cohort. Collected Maka runtime files and egress audit logs are represented in attempt artifacts with -byte counts and SHA-256 digests. +cohort. This URL policy is a blocklist for known benchmark and public-solution contamination +surfaces, not a complete defense against a deliberately invented lookup channel; the network +namespace still forces all subject traffic through the audited proxy. Collected Maka runtime files +and egress audit logs are represented in attempt artifacts with byte counts and SHA-256 digests. +The local image tag remains a machine deployment identity rather than a registry digest; digest +pinning is tracked separately. The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains. diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json index e8f8bd9d0c..b27c2f64d5 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json @@ -99,7 +99,6 @@ "hostSettlementTimeoutMs": 120000, "connectionSlug": "env-deepseek", "baseUrl": "https://api.deepseek.com", - "webTools": "disabled", "model": "deepseek-v4-flash", "thinkingLevel": "max", "permissionMode": "bypass", diff --git a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json index 72cc1fc86c..374eb0d07a 100644 --- a/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json +++ b/packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json @@ -77,7 +77,6 @@ "runtimeHostsPath": "/tmp/maka-runtime-hosts", "connectionSlug": "env-openai", "baseUrl": "https://api.deepseek.com", - "webTools": "disabled", "model": "deepseek-v4-flash", "thinkingLevel": "max", "permissionMode": "bypass", diff --git a/packages/eval/harbor/egress_filter.py b/packages/eval/harbor/egress_filter.py index a429f211b4..fc744672af 100644 --- a/packages/eval/harbor/egress_filter.py +++ b/packages/eval/harbor/egress_filter.py @@ -14,7 +14,7 @@ MAX_AUDIT_BYTES = 1024 * 1024 AUDIT_PATH = Path(os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress/hits.jsonl")) PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})") -TERMINAL_BENCH = re.compile(r"terminal[-_.%/]*bench", re.IGNORECASE) +TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE) def contamination_rule(raw_url: str) -> tuple[str, str, str] | None: diff --git a/packages/eval/harbor/relay_agent.py b/packages/eval/harbor/relay_agent.py index 16b583ab30..9168ee1ab9 100644 --- a/packages/eval/harbor/relay_agent.py +++ b/packages/eval/harbor/relay_agent.py @@ -31,6 +31,8 @@ class RelayTransportClosed(RuntimeError): SCOPE_ERROR_PREFIX = "MAKA-EVAL-SCOPE-ERROR-V1" RESULT_PAYLOAD_LIMIT_BYTES = 2 * 1024 RESULT_CARRIER_LIMIT_BYTES = 64 * 1024 +SUBJECT_STDOUT_PATH = "/logs/artifacts/maka-subject.stdout.txt" +SUBJECT_STDERR_PATH = "/logs/artifacts/maka-subject.stderr.txt" _host_teardown_requested = False @@ -109,6 +111,7 @@ async def run(self, instruction: str, environment: Any, context: Any) -> None: decision.result() raise RelayTransportClosed("Maka Eval relay received control before execution") result = execution.result() + await _persist_subject_outputs(environment, result) stdout, diagnostic = _project_result(result, request) if diagnostic["category"] != "execution-scope-unavailable": await _quiesce_scope(environment, cwd, scope_path) @@ -146,6 +149,8 @@ async def run(self, instruction: str, environment: Any, context: Any) -> None: result = await _settle_or_destroy( environment, cwd, scope_path, execution, self._teardown_timeout ) + if result is not None: + await _persist_subject_outputs(environment, result) if ( result is not None and not execution_reported @@ -184,10 +189,20 @@ async def run(self, instruction: str, environment: Any, context: Any) -> None: raise except RelayTransportClosed: if request is not None and execution is not None: - await _settle_or_destroy(environment, cwd, scope_path, execution, self._teardown_timeout) + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) except BaseException: if request is not None and execution is not None: - await _settle_or_destroy(environment, cwd, scope_path, execution, self._teardown_timeout) + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) raise finally: if decision is not None and not decision.done(): @@ -264,11 +279,25 @@ async def _prepare_command( f"{{ echo $$ > {shlex.quote(scope_path)}; }} 2>/dev/null || " f"{{ printf {scope_error}; exit 111; }}; " f". {shlex.quote(container_path)}; command -p rm -f {shlex.quote(container_path)}; " - f"exec {subject}{output_redirect} 2>/dev/null" + f"exec {subject}{output_redirect}" ) return f"setsid --wait sh -c {shlex.quote(inner)}" +async def _persist_subject_outputs(environment: Any, result: Any) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + stdout = root / "stdout" + stderr = root / "stderr" + stdout.write_text(str(getattr(result, "stdout", "") or ""), encoding="utf-8") + stderr.write_text(str(getattr(result, "stderr", "") or ""), encoding="utf-8") + prepared = await environment.exec("mkdir -p /logs/artifacts && chmod 700 /logs/artifacts") + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare subject artifact output") + await environment.upload_file(stdout, SUBJECT_STDOUT_PATH) + await environment.upload_file(stderr, SUBJECT_STDERR_PATH) + + def _decode_result_carrier(carrier: str, token: str) -> tuple[str, dict[str, Any]]: raw = carrier.encode("utf-8", errors="replace") if len(raw) > RESULT_CARRIER_LIMIT_BYTES: diff --git a/packages/eval/harbor/run_trial.py b/packages/eval/harbor/run_trial.py index 29eb39e021..ee15dde972 100644 --- a/packages/eval/harbor/run_trial.py +++ b/packages/eval/harbor/run_trial.py @@ -133,9 +133,12 @@ async def run_trial(framework: str, expected_version: str, config_file: Path) -> def apply_subject_egress_policy(task: object) -> None: + required = os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") == "1" allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") - if not allowed_host: + if not required: return + if not allowed_host: + raise RuntimeError("required Eval egress proxy host is unavailable") agent = task.config.agent agent.network_mode = "allowlist" agent.allowed_hosts = [allowed_host] diff --git a/packages/eval/harbor/test_egress_filter.py b/packages/eval/harbor/test_egress_filter.py index c9ba4a3bd7..477c704562 100644 --- a/packages/eval/harbor/test_egress_filter.py +++ b/packages/eval/harbor/test_egress_filter.py @@ -28,6 +28,7 @@ def test_blocks_contamination_surfaces_and_recursive_jina_urls(self) -> None: "https://r.jina.ai/https://github.com/harbor-framework/terminal-bench-2-1": "jina_recursive:benchmark_repository", "https://r.jina.ai/https%253A%252F%252Fspylab.ai%252Fterminal-bench": "jina_recursive:terminal_bench_url", "https://example.test/search?q=TeRmInAlBeNcH": "terminal_bench_url", + "https://google.com/search?q=terminal+bench": "terminal_bench_url", "https://github.com/harbor-framework/terminal%252Dbench-2-1.git": "benchmark_repository", } for url, rule_id in blocked.items(): diff --git a/packages/eval/harbor/test_relay_artifacts.py b/packages/eval/harbor/test_relay_artifacts.py new file mode 100644 index 0000000000..2de50d8ef0 --- /dev/null +++ b/packages/eval/harbor/test_relay_artifacts.py @@ -0,0 +1,66 @@ +import asyncio +import importlib +import os +import shutil +import sys +import tempfile +import types +import unittest +from pathlib import Path +from types import SimpleNamespace + + +class BaseAgent: + def __init__(self, *args, **kwargs): + pass + + +def load_relay(): + os.environ["MAKA_EVAL_FRAMEWORK"] = "harbor" + package = types.ModuleType("harbor") + agents = types.ModuleType("harbor.agents") + base = types.ModuleType("harbor.agents.base") + base.BaseAgent = BaseAgent + sys.modules["harbor"] = package + sys.modules["harbor.agents"] = agents + sys.modules["harbor.agents.base"] = base + sys.modules.pop("relay_agent", None) + return importlib.import_module("relay_agent") + + +class ArtifactEnvironment: + def __init__(self, root: Path): + self.root = root + + async def exec(self, command, cwd=None, timeout_sec=None): + return SimpleNamespace(return_code=0, stdout="", stderr="") + + async def upload_file(self, source, target): + destination = self.root / target.lstrip("/") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +class RelayArtifactTest(unittest.IsolatedAsyncioTestCase): + async def test_framed_transport_outputs_are_persisted_without_changing_the_carrier(self): + relay = load_relay() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + environment = ArtifactEnvironment(root) + result = SimpleNamespace(stdout="framed-result\n", stderr="diagnostic\n") + + await relay._persist_subject_outputs(environment, result) + + self.assertEqual( + (root / relay.SUBJECT_STDOUT_PATH.lstrip("/")).read_text(), + "framed-result\n", + ) + self.assertEqual( + (root / relay.SUBJECT_STDERR_PATH.lstrip("/")).read_text(), + "diagnostic\n", + ) + self.assertEqual(result.stdout, "framed-result\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/eval/harbor/test_relay_lifecycle.py b/packages/eval/harbor/test_relay_lifecycle.py index 2557d1dff0..0802056679 100644 --- a/packages/eval/harbor/test_relay_lifecycle.py +++ b/packages/eval/harbor/test_relay_lifecycle.py @@ -22,12 +22,16 @@ def __init__(self, *args, **kwargs): class LocalEnvironment: async def upload_file(self, source, target): + if target.startswith("/logs/"): + return shutil.copyfile(source, target) async def download_file(self, source, target): shutil.copyfile(source, target) async def exec(self, command, cwd=None, timeout_sec=None): + if command.startswith("mkdir -p /logs/artifacts"): + return SimpleNamespace(return_code=0, stdout="", stderr="") completed = await asyncio.to_thread( subprocess.run, command, @@ -47,6 +51,8 @@ def __init__(self): self.finished = asyncio.Event() async def upload_file(self, source, target): + if target.startswith("/logs/"): + return shutil.copyfile(source, target) async def download_file(self, source, target): @@ -115,6 +121,8 @@ async def exec(self, command, cwd=None, timeout_sec=None): ) if command.startswith("rm -f --"): return SimpleNamespace(return_code=0, stdout="", stderr="") + if command.startswith("mkdir -p /logs/artifacts"): + return SimpleNamespace(return_code=0, stdout="", stderr="") raise AssertionError(f"unexpected command after scope setup failure: {command}") diff --git a/packages/eval/harbor/test_run_trial_policy.py b/packages/eval/harbor/test_run_trial_policy.py index 4f251541a6..762a3e35d2 100644 --- a/packages/eval/harbor/test_run_trial_policy.py +++ b/packages/eval/harbor/test_run_trial_policy.py @@ -18,13 +18,22 @@ def test_forces_only_the_subject_phase_through_the_cell_proxy(self) -> None: task = SimpleNamespace(config=SimpleNamespace(agent=agent)) with patch.dict( os.environ, - {"MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy"}, + { + "MAKA_EVAL_EGRESS_REQUIRED": "1", + "MAKA_EVAL_EGRESS_ALLOWED_HOST": "maka-eval-mitmproxy", + }, clear=False, ): MODULE.apply_subject_egress_policy(task) self.assertEqual(agent.network_mode, "allowlist") self.assertEqual(agent.allowed_hosts, ["maka-eval-mitmproxy"]) + def test_required_egress_fails_closed_without_the_proxy_host(self) -> None: + task = SimpleNamespace(config=SimpleNamespace(agent=SimpleNamespace())) + with patch.dict(os.environ, {"MAKA_EVAL_EGRESS_REQUIRED": "1"}, clear=True): + with self.assertRaisesRegex(RuntimeError, "proxy host is unavailable"): + MODULE.apply_subject_egress_policy(task) + if __name__ == "__main__": unittest.main() diff --git a/packages/eval/package.json b/packages/eval/package.json index 8ed46892bc..98b01a2d19 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -19,7 +19,7 @@ "clean": "node ../../scripts/clean-paths.mjs dist tsconfig.tsbuildinfo", "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py && python3 harbor/test_egress_filter.py && python3 harbor/test_run_trial_policy.py" + "test:dist": "node --test \"dist/**/*.test.js\" && python3 harbor/test_relay_contract.py && python3 harbor/test_relay_lifecycle.py && python3 harbor/test_egress_filter.py && python3 harbor/test_run_trial_policy.py && python3 harbor/test_relay_artifacts.py" }, "dependencies": { "@maka/core": "0.1.0", diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts index d2f3dd23bb..fd4a024c9d 100644 --- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts +++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts @@ -9,7 +9,7 @@ import { FileAttemptStore } from '../attempt-store.js'; import type { ExperimentCell, ExperimentSpec, JsonObject } from '../experiment.js'; import { createExternalSubjectAdapter } from '../external-subject.js'; import { createHarborExecutor } from '../harness-executor.js'; -import { disabledWebToolsRuntimePolicyDocument } from '../maka-runtime-policy.js'; +import { makaEvalRuntimePolicyDocument } from '../maka-runtime-policy.js'; import { createMakaSubjectAdapter } from '../maka-subject.js'; import { runExperiment, @@ -780,7 +780,7 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', ) as { subjects: Array<{ id: string; - config: { args?: string[]; webTools?: string; toolProfile?: string }; + config: { args?: string[]; toolProfile?: string }; }>; execution: { maxConcurrentTaskGroups: number }; executor: { @@ -836,7 +836,6 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', const pi = spec.subjects.find(({ id }) => id === 'pi')!; const maka = spec.subjects.find(({ id }) => id === 'maka')!; const zcode = spec.subjects.find(({ id }) => id === 'zcode')!; - assert.equal(maka.config.webTools, 'disabled'); assert.equal( (maka.config as { hostSettlementTimeoutMs?: number }).hostSettlementTimeoutMs, 120_000, @@ -857,9 +856,8 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract', assert.equal(pi.config.args?.includes('max'), true); }); -test('Maka web-tool policy removes both hosted search and fetch surfaces', () => { - const document = disabledWebToolsRuntimePolicyDocument(); - assert.equal(document.policy.webSearch.enabled, false); +test('Maka Eval policy enables privacy independently of the tool profile', () => { + const document = makaEvalRuntimePolicyDocument(); assert.equal(document.policy.privacy.incognitoActive, true); }); @@ -1042,7 +1040,6 @@ function makaConfig() { shimPath: '/opt/maka/harbor-maka-subject.js', runtimeHostsPath: '/tmp/maka-runtime-hosts', baseUrl: 'https://provider.test/v1', - webTools: 'disabled', connectionSlug: 'provider', model: 'deepseek-v4-flash', thinkingLevel: 'max', diff --git a/packages/eval/src/__tests__/maka-artifacts.test.ts b/packages/eval/src/__tests__/maka-artifacts.test.ts new file mode 100644 index 0000000000..9eb9a1e824 --- /dev/null +++ b/packages/eval/src/__tests__/maka-artifacts.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import test from 'node:test'; +import { + captureMakaRuntimeArtifacts, + MAKA_RUNTIME_ARTIFACT_PATH, + MAKA_SUBJECT_STDERR_PATH, + MAKA_SUBJECT_STDOUT_PATH, +} from '../maka-artifacts.js'; +import { createMakaSubjectAdapter } from '../maka-subject.js'; +import type { ExperimentCell } from '../experiment.js'; +import type { SubjectExecutionContext } from '../runner.js'; + +test('runtime artifact capture includes committed WAL rows in a standalone database', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-eval-artifacts-')); + const stateRoot = join(root, 'state'); + const destinationRoot = join(root, 'artifacts'); + await mkdir(stateRoot); + const database = new DatabaseSync(join(stateRoot, 'runtime.sqlite')); + try { + database.exec('PRAGMA journal_mode=WAL'); + database.exec('PRAGMA wal_autocheckpoint=0'); + database.exec('CREATE TABLE evidence (value TEXT NOT NULL)'); + database.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + database.prepare('INSERT INTO evidence (value) VALUES (?)').run('from-wal'); + await writeFile(join(stateRoot, 'runtime-host-candidate.log'), 'candidate-ready\n', { + mode: 0o600, + }); + + const manifest = await captureMakaRuntimeArtifacts({ + stateRoot, + destinationRoot, + reason: 'signal', + now: () => 42, + }); + + assert.equal(manifest.capturedAt, 42); + assert.equal(manifest.reason, 'signal'); + const snapshot = new DatabaseSync(join(destinationRoot, 'runtime.sqlite'), { readOnly: true }); + try { + assert.equal(snapshot.prepare('SELECT value FROM evidence').get()?.value, 'from-wal'); + } finally { + snapshot.close(); + } + await assert.rejects(stat(join(destinationRoot, 'runtime.sqlite-wal')), { code: 'ENOENT' }); + await assert.rejects(stat(join(destinationRoot, 'runtime.sqlite-shm')), { code: 'ENOENT' }); + assert.equal( + await readFile(join(destinationRoot, 'runtime-host-candidate.log'), 'utf8'), + 'candidate-ready\n', + ); + const databaseFile = manifest.files.find(({ path }) => path === 'runtime.sqlite'); + assert.ok(databaseFile); + assert.equal( + databaseFile.sha256, + `sha256:${createHash('sha256') + .update(await readFile(join(destinationRoot, 'runtime.sqlite'))) + .digest('hex')}`, + ); + } finally { + database.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('Maka reports the runtime and process artifacts for settled and timed-out executions', async () => { + for (const termination of ['exited', 'framework_timeout'] as const) { + const result = await createMakaSubjectAdapter().execute({ + cell: makaCell(), + context: { + cwd: '/app', + taskInput: 'solve', + metadata: {}, + execute: async (input: Parameters[0]) => { + const payload = JSON.parse(Buffer.from(input.args[1] ?? '', 'base64url').toString()) as { + artifactRoot: string; + execution: { executionId: string }; + }; + assert.equal(payload.artifactRoot, MAKA_RUNTIME_ARTIFACT_PATH); + return { + termination, + exitCode: termination === 'exited' ? 0 : 124, + stdout: JSON.stringify({ + executionId: payload.execution.executionId, + kind: 'settled', + status: termination === 'exited' ? 'completed' : 'cancelled', + usage: { + inputTokens: 1, + outputTokens: 2, + cacheReadTokens: 0, + cacheWriteTokens: 0, + reasoningTokens: 0, + totalTokens: 3, + }, + costUsd: null, + }), + diagnostic: { category: 'none' }, + }; + }, + }, + }); + assert.equal(result.status, termination === 'exited' ? 'completed' : 'failed'); + assert.deepEqual( + result.artifacts.map(({ kind, path }) => ({ kind, path })), + [ + { kind: 'maka-runtime-state', path: MAKA_RUNTIME_ARTIFACT_PATH }, + { kind: 'subject-stdout', path: MAKA_SUBJECT_STDOUT_PATH }, + { kind: 'subject-stderr', path: MAKA_SUBJECT_STDERR_PATH }, + ], + ); + } +}); + +function makaCell(): ExperimentCell { + return { + id: 'task::1::maka', + experimentId: 'experiment', + benchmark: { id: 'benchmark', version: 'revision', config: {} }, + executor: { kind: 'harbor', config: {} }, + subject: { + id: 'maka', + kind: 'maka', + credentials: ['DEEPSEEK_API_KEY'], + config: { + nodePath: '/opt/node', + shimPath: '/opt/maka-subject.js', + runtimeHostsPath: '/tmp/maka-runtime-hosts', + baseUrl: 'https://api.deepseek.com', + connectionSlug: 'env-deepseek', + model: 'deepseek-v4-flash', + thinkingLevel: 'max', + permissionMode: 'bypass', + collaborationMode: 'agent', + orchestrationMode: 'default', + hostSettlementTimeoutMs: 120_000, + toolProfile: 'headless-coding-v1', + }, + }, + task: { id: 'task', input: 'solve', config: {} }, + repetition: 1, + budget: { timeoutMultiplier: 1, maxSteps: 100 }, + verifier: { reward: 'reward' }, + }; +} diff --git a/packages/eval/src/harbor-maka-subject.ts b/packages/eval/src/harbor-maka-subject.ts index 447abb7e47..0fdee1d24d 100644 --- a/packages/eval/src/harbor-maka-subject.ts +++ b/packages/eval/src/harbor-maka-subject.ts @@ -2,40 +2,65 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { runHostedExecution } from '@maka/runtime-host/client'; import type { HostedExecutionStartInput } from '@maka/runtime-host/protocol'; -import { disabledWebToolsRuntimePolicyDocument } from './maka-runtime-policy.js'; +import { captureMakaRuntimeArtifacts, writeMakaArtifactCollectionError } from './maka-artifacts.js'; +import { makaEvalRuntimePolicyDocument } from './maka-runtime-policy.js'; import { takeRelayResultToken, writeRelayResult } from './relay-result-frame.js'; const resultToken = takeRelayResultToken(); const payload = JSON.parse(Buffer.from(process.argv[2] ?? '', 'base64url').toString()) as { rootPath: string; + artifactRoot: string; baseUrl: string; - webTools: 'enabled' | 'disabled'; hostSettlementTimeoutMs: number; execution: HostedExecutionStartInput; }; const abort = new AbortController(); -process.once('SIGINT', () => abort.abort()); -process.once('SIGTERM', () => abort.abort()); +let artifactCapture = Promise.resolve(); +const captureArtifacts = (reason: 'settled' | 'signal') => { + artifactCapture = artifactCapture.then(async () => { + try { + await captureMakaRuntimeArtifacts({ + stateRoot: payload.rootPath, + destinationRoot: payload.artifactRoot, + reason, + }); + } catch (error) { + await writeMakaArtifactCollectionError(payload.artifactRoot, error).catch(() => undefined); + } + }); + return artifactCapture; +}; +const stop = () => { + abort.abort(); + void captureArtifacts('signal'); +}; +process.once('SIGINT', stop); +process.once('SIGTERM', stop); const runtimeHome = join(dirname(payload.rootPath), `${process.pid}-home`); await mkdir(payload.rootPath, { recursive: true }); await mkdir(runtimeHome, { recursive: true, mode: 0o700 }); -if (payload.webTools === 'disabled') { - await writeFile( - join(payload.rootPath, 'runtime-policy.json'), - `${JSON.stringify(disabledWebToolsRuntimePolicyDocument(process.env.HTTPS_PROXY))}\n`, - { flag: 'wx', mode: 0o600 }, - ); -} +await writeFile( + join(payload.rootPath, 'runtime-policy.json'), + `${JSON.stringify(makaEvalRuntimePolicyDocument(process.env.HTTPS_PROXY))}\n`, + { flag: 'wx', mode: 0o600 }, +); process.env.HOME = runtimeHome; process.env.DEEPSEEK_BASE_URL = payload.baseUrl; -const result = await runHostedExecution({ - rootPath: payload.rootPath, - baseUrl: payload.baseUrl, - execution: payload.execution, - signal: abort.signal, - hostSettlementTimeoutMs: payload.hostSettlementTimeoutMs, -}); +let result: Awaited>; +try { + result = await runHostedExecution({ + rootPath: payload.rootPath, + baseUrl: payload.baseUrl, + execution: payload.execution, + signal: abort.signal, + hostSettlementTimeoutMs: payload.hostSettlementTimeoutMs, + }); +} finally { + await captureArtifacts(abort.signal.aborted ? 'signal' : 'settled'); + process.removeListener('SIGINT', stop); + process.removeListener('SIGTERM', stop); +} const failureReason = result.failureReason; const framedResult = failureReason !== undefined && Buffer.byteLength(failureReason) > 768 diff --git a/packages/eval/src/harness-executor.ts b/packages/eval/src/harness-executor.ts index fd9fb3fb41..7005793725 100644 --- a/packages/eval/src/harness-executor.ts +++ b/packages/eval/src/harness-executor.ts @@ -1,12 +1,18 @@ import { spawn, type ChildProcess } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { once } from 'node:events'; -import { chmod, mkdir, readFile, unlink, writeFile } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { chmod, lstat, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises'; import { createServer, type Server, type Socket } from 'node:net'; -import { delimiter, dirname, join, relative, resolve, sep } from 'node:path'; +import { basename, delimiter, dirname, join, relative, resolve, sep } from 'node:path'; import { createInterface } from 'node:readline'; import { fileURLToPath } from 'node:url'; import { decodeJsonObject, type ExperimentCell, type JsonObject } from './experiment.js'; +import { + MAKA_RUNTIME_ARTIFACT_PATH, + MAKA_SUBJECT_STDERR_PATH, + MAKA_SUBJECT_STDOUT_PATH, +} from './maka-artifacts.js'; import { type ExecutorAttemptOutcome, type ExperimentExecutor, @@ -615,7 +621,12 @@ function preparationEnvironment( ...inherited, MAKA_EVAL_FRAMEWORK: framework, PYTHONPATH: [relayPath, inherited.PYTHONPATH].filter(Boolean).join(delimiter), - ...(egressAllowedHost ? { MAKA_EVAL_EGRESS_ALLOWED_HOST: egressAllowedHost } : {}), + ...(egressAllowedHost + ? { + MAKA_EVAL_EGRESS_REQUIRED: '1', + MAKA_EVAL_EGRESS_ALLOWED_HOST: egressAllowedHost, + } + : {}), ...(networkPolicyPath ? { MAKA_EVAL_NETWORK_POLICY_PATH: networkPolicyPath } : {}), }; } @@ -686,6 +697,7 @@ async function readVerification( failureReason: score === null ? 'verifier produced no reward' : null, artifacts: [ { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName }, + ...(await collectedArtifactInventory(state.trialPath)), ...(egressAudit ? [ { @@ -700,6 +712,48 @@ async function readVerification( }; } +async function collectedArtifactInventory(trialPath: string): Promise { + const root = join(trialPath, 'artifacts', 'logs', 'artifacts'); + const files: JsonObject[] = []; + const targets = [ + join(root, basename(MAKA_RUNTIME_ARTIFACT_PATH)), + join(root, basename(MAKA_SUBJECT_STDOUT_PATH)), + join(root, basename(MAKA_SUBJECT_STDERR_PATH)), + ]; + for (const target of targets) { + await walkCollectedArtifacts(trialPath, target, files).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + }); + } + return files.sort((left, right) => String(left.path).localeCompare(String(right.path))); +} + +async function walkCollectedArtifacts( + trialPath: string, + current: string, + files: JsonObject[], +): Promise { + const metadata = await lstat(current); + if (metadata.isSymbolicLink()) return; + if (metadata.isFile()) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(current)) hash.update(chunk as Buffer); + files.push({ + kind: 'collected-artifact', + path: relative(trialPath, current).split(sep).join('/'), + bytes: metadata.size, + sha256: `sha256:${hash.digest('hex')}`, + }); + return; + } + if (!metadata.isDirectory()) return; + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) continue; + await walkCollectedArtifacts(trialPath, path, files); + } +} + interface HarnessOptions { readonly frameworkVersion: string; readonly pythonPathEnv: string; diff --git a/packages/eval/src/maka-artifacts.ts b/packages/eval/src/maka-artifacts.ts new file mode 100644 index 0000000000..d522e55b81 --- /dev/null +++ b/packages/eval/src/maka-artifacts.ts @@ -0,0 +1,135 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { chmod, copyFile, mkdir, rename, rm, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; +import { backup, DatabaseSync } from 'node:sqlite'; + +export const MAKA_RUNTIME_ARTIFACT_PATH = '/logs/artifacts/maka-runtime-host'; +export const MAKA_SUBJECT_STDOUT_PATH = '/logs/artifacts/maka-subject.stdout.txt'; +export const MAKA_SUBJECT_STDERR_PATH = '/logs/artifacts/maka-subject.stderr.txt'; + +interface CapturedFile { + readonly path: string; + readonly bytes: number; + readonly sha256: `sha256:${string}`; +} + +export interface MakaRuntimeArtifactManifest { + readonly schemaVersion: 'maka.eval.runtime_artifacts.v1'; + readonly reason: 'settled' | 'signal'; + readonly capturedAt: number; + readonly files: readonly CapturedFile[]; +} + +export async function captureMakaRuntimeArtifacts(input: { + readonly stateRoot: string; + readonly destinationRoot: string; + readonly reason: MakaRuntimeArtifactManifest['reason']; + readonly now?: () => number; +}): Promise { + const stateRoot = resolve(input.stateRoot); + const destinationRoot = resolve(input.destinationRoot); + const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; + await rm(stagingRoot, { recursive: true, force: true }); + try { + await mkdir(stagingRoot, { recursive: true, mode: 0o700 }); + const sourcePath = join(stateRoot, 'runtime.sqlite'); + const destinationPath = join(stagingRoot, 'runtime.sqlite'); + const source = new DatabaseSync(sourcePath, { readOnly: true }); + try { + await backup(source, destinationPath); + } finally { + source.close(); + } + const snapshot = new DatabaseSync(destinationPath); + try { + snapshot.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + snapshot.exec('PRAGMA journal_mode=DELETE'); + } finally { + snapshot.close(); + } + await rm(`${destinationPath}-wal`, { force: true }); + await rm(`${destinationPath}-shm`, { force: true }); + await chmod(destinationPath, 0o600); + + for (const name of ['runtime-host-candidate.log', 'runtime-policy.json']) { + const source = join(stateRoot, name); + try { + const metadata = await stat(source); + if (!metadata.isFile()) continue; + const destination = join(stagingRoot, name); + await copyFile(source, destination); + await chmod(destination, 0o600); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + } + + const files = await Promise.all( + ['runtime.sqlite', 'runtime-host-candidate.log', 'runtime-policy.json'].map((name) => + describeFile(join(stagingRoot, name), name), + ), + ); + const manifest: MakaRuntimeArtifactManifest = { + schemaVersion: 'maka.eval.runtime_artifacts.v1', + reason: input.reason, + capturedAt: (input.now ?? Date.now)(), + files: files.filter((file): file is CapturedFile => file !== null), + }; + await writeFile(join(stagingRoot, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, { + flag: 'wx', + mode: 0o600, + }); + await mkdir(dirname(destinationRoot), { recursive: true, mode: 0o700 }); + await rm(destinationRoot, { recursive: true, force: true }); + await rename(stagingRoot, destinationRoot); + return manifest; + } catch (error) { + await rm(stagingRoot, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +export async function writeMakaArtifactCollectionError( + destinationRoot: string, + error: unknown, +): Promise { + const root = resolve(destinationRoot); + await mkdir(root, { recursive: true, mode: 0o700 }); + await writeFile( + join(root, 'collection-error.json'), + `${JSON.stringify({ + schemaVersion: 'maka.eval.runtime_artifact_error.v1', + errorCode: safeErrorCode(error), + message: safeErrorMessage(error), + })}\n`, + { mode: 0o600 }, + ); +} + +async function describeFile(path: string, name: string): Promise { + try { + const metadata = await stat(path); + if (!metadata.isFile()) return null; + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); + return { + path: basename(name), + bytes: metadata.size, + sha256: `sha256:${hash.digest('hex')}`, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +function safeErrorCode(error: unknown): string | null { + const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined; + return typeof code === 'string' && /^[A-Z0-9_]{1,64}$/u.test(code) ? code : null; +} + +function safeErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return Buffer.from(message).subarray(0, 1024).toString(); +} diff --git a/packages/eval/src/maka-runtime-policy.ts b/packages/eval/src/maka-runtime-policy.ts index 2ae30abd28..abdfb2dddc 100644 --- a/packages/eval/src/maka-runtime-policy.ts +++ b/packages/eval/src/maka-runtime-policy.ts @@ -1,6 +1,6 @@ import { createDefaultRuntimePolicy } from '@maka/core/runtime-policy'; -export function disabledWebToolsRuntimePolicyDocument(proxyUrl?: string) { +export function makaEvalRuntimePolicyDocument(proxyUrl?: string) { const policy = createDefaultRuntimePolicy(); const proxy = proxyUrl ? new URL(proxyUrl) : undefined; return { @@ -20,7 +20,6 @@ export function disabledWebToolsRuntimePolicyDocument(proxyUrl?: string) { } : {}), privacy: { incognitoActive: true }, - webSearch: { ...policy.webSearch, enabled: false }, }, }; } diff --git a/packages/eval/src/maka-subject.ts b/packages/eval/src/maka-subject.ts index dc0da5c621..86fb532991 100644 --- a/packages/eval/src/maka-subject.ts +++ b/packages/eval/src/maka-subject.ts @@ -1,10 +1,15 @@ import { randomUUID } from 'node:crypto'; +import { isSessionToolProfile, type SessionToolProfile } from '@maka/core/session'; import { decodeHostedExecutionProjection, - HOSTED_EXECUTION_TOOL_PROFILES, type HostedExecutionStartInput, } from '@maka/runtime-host/protocol'; import type { JsonObject } from './experiment.js'; +import { + MAKA_RUNTIME_ARTIFACT_PATH, + MAKA_SUBJECT_STDERR_PATH, + MAKA_SUBJECT_STDOUT_PATH, +} from './maka-artifacts.js'; import type { NormalizedUsage } from './result.js'; import type { SubjectAdapter, SubjectExecutionContext } from './runner.js'; @@ -28,16 +33,16 @@ export function createMakaSubjectAdapter(): SubjectAdapter { permissionMode: config.permissionMode, collaborationMode: config.collaborationMode, orchestrationMode: config.orchestrationMode, + toolProfile: config.toolProfile, }, content: { text: context.taskInput }, maxSteps: positive(cell.budget.maxSteps, 'budget.maxSteps'), - toolProfile: config.toolProfile, }; const payload = Buffer.from( JSON.stringify({ rootPath: `${config.runtimeHostsPath}/${executionId}`, + artifactRoot: MAKA_RUNTIME_ARTIFACT_PATH, baseUrl: config.baseUrl, - webTools: config.webTools, hostSettlementTimeoutMs: config.hostSettlementTimeoutMs, execution: input, }), @@ -66,7 +71,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter { durationMs: Date.now() - startedAt, status: 'failed' as const, failureReason: 'Maka subject exceeded the framework timeout', - artifacts: [], + artifacts: makaArtifacts(executionId, process), }; } if (process.stdout.length === 0) { @@ -97,7 +102,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter { projection.failureReason, 'Maka execution did not settle', ), - artifacts: [], + artifacts: makaArtifacts(executionId, process), }; } const result = ( @@ -109,7 +114,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter { durationMs: Date.now() - startedAt, status, failureReason, - artifacts: [], + artifacts: makaArtifacts(executionId, process), }); if (process.exitCode !== 0) { return result('indeterminate', 'Maka execution shim did not settle cleanly'); @@ -184,6 +189,7 @@ function subjectFailure( status: cancelled ? ('indeterminate' as const) : ('infra_failed' as const), failureReason: cancelled ? 'Maka subject cancelled' : `Maka subject failed during ${stage}`, artifacts: [ + ...makaArtifacts(undefined, process), { kind: 'subject-failure', stage, @@ -200,20 +206,41 @@ function subjectFailure( }; } +function makaArtifacts( + executionId: string | undefined, + process?: Awaited>, +): JsonObject[] { + return [ + { + kind: 'maka-runtime-state', + path: MAKA_RUNTIME_ARTIFACT_PATH, + ...(executionId ? { executionId } : {}), + }, + { + kind: 'subject-stdout', + path: MAKA_SUBJECT_STDOUT_PATH, + ...(process ? { bytes: Buffer.byteLength(process.stdout) } : {}), + }, + { + kind: 'subject-stderr', + path: MAKA_SUBJECT_STDERR_PATH, + }, + ]; +} + interface MakaConfig { readonly nodePath: string; readonly shimPath: string; readonly runtimeHostsPath: string; readonly hostSettlementTimeoutMs: number; readonly baseUrl: string; - readonly webTools: 'enabled' | 'disabled'; readonly connectionSlug: string; readonly model: string; readonly thinkingLevel: HostedExecutionStartInput['session']['thinkingLevel']; readonly permissionMode: HostedExecutionStartInput['session']['permissionMode']; readonly collaborationMode: HostedExecutionStartInput['session']['collaborationMode']; readonly orchestrationMode: HostedExecutionStartInput['session']['orchestrationMode']; - readonly toolProfile: NonNullable; + readonly toolProfile: SessionToolProfile; } function decodeConfig(value: JsonObject): MakaConfig { @@ -231,21 +258,16 @@ function decodeConfig(value: JsonObject): MakaConfig { 'hostSettlementTimeoutMs', 'toolProfile', ]; - if (Object.hasOwn(value, 'webTools')) fields.push('webTools'); const config = exact(value, fields); if (!URL.canParse(String(config.baseUrl))) throw new Error('Maka baseUrl is invalid'); - const webTools = config.webTools ?? 'enabled'; - if (webTools !== 'enabled' && webTools !== 'disabled') { - throw new Error('Maka config.webTools is invalid'); - } const hostSettlementTimeoutMs = positiveInteger( config.hostSettlementTimeoutMs, 'Maka config.hostSettlementTimeoutMs', ); - if (!(HOSTED_EXECUTION_TOOL_PROFILES as readonly unknown[]).includes(config.toolProfile)) { + if (!isSessionToolProfile(config.toolProfile)) { throw new Error('Maka config.toolProfile is invalid'); } - return { ...config, webTools, hostSettlementTimeoutMs } as unknown as MakaConfig; + return { ...config, hostSettlementTimeoutMs } as unknown as MakaConfig; } function exact(value: unknown, fields: readonly string[]): Record { diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 8bfa090ab0..81a49ba6b9 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -77,6 +77,7 @@ import { HostClientCapabilityCoordinator } from '../server/client-capability-coo import type { HostMemoryCoordinator } from '../server/memory-coordinator.js'; import type { ConnectionContext } from '../server/operation-dispatcher.js'; import { RuntimePolicyActivationGate } from '../server/runtime-policy-activation-gate.js'; +import { HostResidencyRegistry } from '../server/host-residency-registry.js'; import { HostOAuthExecutionAuthority, OAuthExecutionCredentialError, @@ -95,6 +96,10 @@ const MAX_IMPLEMENTATION_CHILD_PTY_READS = 5; const MIN_IMPLEMENTATION_CHILD_REQUESTS = 6; const MAX_IMPLEMENTATION_CHILD_REQUESTS = MIN_IMPLEMENTATION_CHILD_REQUESTS + MAX_IMPLEMENTATION_CHILD_PTY_READS - 1; +const HEADLESS_CODING_V1_PROMPT_HASH = + 'sha256:0e3389e330b8b8f0db1c7a8b8e2126325fe4c672d6eff279afcd3f9412e52271'; +const HEADLESS_CODING_V1_TOOLS_HASH = + 'sha256:ea1f293096e5e209ae49346f46b0e8ff9b54ae17452a5a23149ad7233afaeafc'; const execFileAsync = promisify(execFile); test('backend creation aborts a stalled canonical connection read', async () => { @@ -662,6 +667,159 @@ test('production backend preserves coordinator Client Capability semantics acros } }); +test('hosted execution freezes the headless coding provider wire contract', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-hosted-profile-wire-')); + const root = join(base, 'interactive'); + const provider = await startProvider(); + const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) return; + const residencies = new HostResidencyRegistry(); + const context: ConnectionContext = { + hostEpoch: 'hosted-profile-wire-epoch', + connectionId: 'hosted-profile-wire-client', + surface: 'run', + principal: 'runtime_host', + acquireResidency: () => residencies.acquire('hosted-profile-wire-operation'), + }; + let composition: Awaited> | undefined; + try { + const policy = await openInteractiveRuntimePolicyStoresForWrite(owner.lease); + const created = await policy.connectionCatalog.create({ + expectedCatalogRevision: 0, + connection: { + slug: 'profile-deepseek', + name: 'Profile DeepSeek', + providerType: 'deepseek', + baseUrl: provider.baseUrl, + enabled: true, + enabledModelIds: ['deepseek-v4-flash'], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') return; + const connection = created.snapshot.connections[0]; + assert.ok(connection); + if (!connection) return; + const configured = await policy.credentialVault.set({ + locator: { + scope: 'connection', + connectionId: connection.connectionId, + kind: 'api_key', + }, + expected: null, + secret: API_KEY, + }); + assert.equal(configured.kind, 'committed'); + await publishConnectionModel(policy, connection.connectionId, 'deepseek-v4-flash'); + + composition = await createExecutionRuntimeHostComposition( + { + owner, + hostEpoch: context.hostEpoch, + acquireResidency: (label) => residencies.acquire(label), + retainUntilProcessExit: () => undefined, + requestDrain: () => composition?.beginDrain(), + waitForResidencies: () => residencies.waitForEmpty(), + waitForResidenciesExcept: (label) => residencies.waitForEmptyExcept(label), + }, + { bootstrapRuntimePolicy: false }, + ); + await composition.recover(); + const executionId = '00000000-0000-4000-8000-000000000777'; + const outcome = await composition.handlers['hosted.execution.start']( + { + executionId, + session: { + workspace: { kind: 'host_path', path: root }, + modelTarget: { + kind: 'explicit', + connectionSlug: 'profile-deepseek', + model: 'deepseek-v4-flash', + }, + permissionMode: 'bypass', + collaborationMode: 'agent', + orchestrationMode: 'default', + toolProfile: 'headless-coding-v1', + }, + content: { text: 'Complete the benchmark task.' }, + maxSteps: 100_000, + }, + context, + ); + assert.equal(outcome.ok, true); + if (!outcome.ok) return; + assert.equal(outcome.result.kind, 'settled'); + + const request = provider.requests.find( + (candidate) => candidate.url === '/v1/responses' && Array.isArray(candidate.body.tools), + ); + assert.ok(request); + const instructions = responsesDeveloperPrompt(request?.body); + const tools = request?.body.tools; + assert.equal(typeof instructions, 'string', JSON.stringify(request?.body)); + assert.ok(Array.isArray(tools)); + assert.equal(stableHash(instructions), HEADLESS_CODING_V1_PROMPT_HASH); + assert.equal(stableHash(tools), HEADLESS_CODING_V1_TOOLS_HASH); + assert.deepEqual(responsesToolNames(request?.body), [ + 'ArchiveRead', + 'Bash', + 'Glob', + 'Grep', + 'Read', + 'apply_patch', + ]); + const bash = (tools as Array>).find((tool) => tool.name === 'Bash'); + assert.ok(bash); + assert.doesNotMatch(JSON.stringify(bash), /run_in_background|pty/u); + + const stores = await openInteractiveExecutionStoresForWrite(owner.lease); + assert.equal( + (await stores.sessionStore.readHeaderSnapshot(executionId)).toolProfile, + 'headless-coding-v1', + ); + const secondTurnId = '00000000-0000-4000-8000-000000000778'; + const secondStarted = await startTurn( + composition, + executionId, + secondTurnId, + 'Continue the benchmark task.', + context, + ); + const secondTerminal = await waitForTerminal( + composition, + executionId, + secondTurnId, + secondStarted, + context, + ); + assert.equal(secondTerminal.status, 'completed'); + const profiledRequests = provider.requests.filter( + (candidate) => candidate.url === '/v1/responses' && Array.isArray(candidate.body.tools), + ); + assert.equal(profiledRequests.length, 2); + for (const profiled of profiledRequests) { + assert.equal( + stableHash(responsesDeveloperPrompt(profiled.body)), + HEADLESS_CODING_V1_PROMPT_HASH, + ); + assert.equal(stableHash(profiled.body.tools), HEADLESS_CODING_V1_TOOLS_HASH); + } + } finally { + try { + await composition?.close(); + } finally { + try { + await owner.close(); + } finally { + await provider.close(); + await rm(base, { recursive: true, force: true }); + } + } + } +}); + test('production Host executes a canonical ai-sdk Session against a real provider wire', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-host-real-model-')); const root = join(base, 'interactive'); @@ -2366,16 +2524,23 @@ test('a bound tool ceiling excludes dynamic Client Capability tools', () => { ); }); -test('the headless coding profile binds the exact Eval tool ceiling', () => { +test('the headless coding profile freezes the Eval prompt and tool ceiling', async () => { const composition = createInteractiveRunComposer({ runtimePolicy: { revision: 0, policy: createDefaultRuntimePolicy() }, skills: { - readCanonicalModelInventory: async () => ({ inventory: [] }), + readCanonicalModelInventory: async () => { + throw new Error('Profiled prompt must not read the product Skill catalog'); + }, } as unknown as HostSkillCatalogCoordinator, - memory: {} as HostMemoryCoordinator, + memory: { + readPromptProjection: async () => { + throw new Error('Profiled prompt must not read product Memory'); + }, + } as unknown as HostMemoryCoordinator, taskLedger: {} as TaskLedgerStore, builtinTools: {}, boundToolNames: hostedExecutionToolNames('headless-coding-v1'), + toolProfile: 'headless-coding-v1', parentAgentTools: buildParentAgentTools(), scheduledTaskTool: { name: 'ScheduledTask', @@ -2390,6 +2555,22 @@ test('the headless coding profile binds the exact Eval tool ceiling', () => { ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'apply_patch'], ); assert.deepEqual(composition.toolAvailability.groups, []); + assert.equal( + ( + await composition.resolveSystemPrompt({ + sessionId: 'profiled-session', + turnId: 'profiled-turn', + cwd: '/workspace', + workspaceRoot: '/workspace', + }) + ).text, + [ + 'Complete the task by acting with the available tools, not by narrating.', + 'Prefer Read, Glob, and Grep for inspection, Edit and Write for file changes, and Bash for shell commands and tests.', + 'Verify the result when practical.', + 'Stop when the task is complete.', + ].join('\n'), + ); }); function skillFixture(id: string, description: string, content: string): ScannedSkill { @@ -2798,6 +2979,26 @@ function toolNames(body: Record | undefined): string[] { .sort(); } +function responsesToolNames(body: Record | undefined): string[] { + const tools = Array.isArray(body?.tools) ? body.tools : []; + return tools + .flatMap((tool) => { + if (!tool || typeof tool !== 'object') return []; + const name = (tool as { name?: unknown }).name; + return typeof name === 'string' ? [name] : []; + }) + .sort(); +} + +function responsesDeveloperPrompt(body: Record | undefined): string | undefined { + const input = Array.isArray(body?.input) ? body.input : []; + const developer = input.find( + (message): message is Record => + Boolean(message) && typeof message === 'object' && message.role === 'developer', + ); + return typeof developer?.content === 'string' ? developer.content : undefined; +} + function providerRequestTrace(requests: readonly ProviderRequest[]): readonly unknown[] { return requests.map((request) => { return { @@ -2948,6 +3149,10 @@ async function handleProviderRequest( customHeader: request.headers['x-maka-test'] as string | undefined, body, }); + if (request.url === '/v1/responses') { + respondProviderResponsesText(response, RESPONSE_TEXT); + return; + } if (body.stream !== true) { response.writeHead(200, { 'content-type': 'application/json' }); response.end( @@ -3120,6 +3325,67 @@ async function handleProviderRequest( respondProviderText(response, RESPONSE_TEXT); } +function respondProviderResponsesText(response: ServerResponse, text: string): void { + const responseId = 'resp-hosted-profile'; + const messageId = 'msg-hosted-profile'; + const events = [ + { + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: 1, + model: 'deepseek-v4-flash', + status: 'in_progress', + output: [], + }, + }, + { + type: 'response.output_item.added', + output_index: 0, + item: { + type: 'message', + id: messageId, + status: 'in_progress', + role: 'assistant', + content: [], + }, + }, + { + type: 'response.output_text.delta', + content_index: 0, + delta: text, + item_id: messageId, + output_index: 0, + }, + { + type: 'response.output_item.done', + output_index: 0, + item: { + type: 'message', + id: messageId, + status: 'completed', + role: 'assistant', + content: [{ type: 'output_text', text, annotations: [] }], + }, + }, + { + type: 'response.completed', + response: { + id: responseId, + object: 'response', + created_at: 1, + model: 'deepseek-v4-flash', + status: 'completed', + output: [], + usage: { input_tokens: 11, output_tokens: 5, total_tokens: 16 }, + }, + }, + ]; + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end(`${events.map((event) => `data: ${JSON.stringify(event)}`).join('\n\n')}\n\n`); +} + function respondProviderText(response: ServerResponse, text: string): void { response.writeHead(200, { 'content-type': 'text/event-stream' }); response.write( diff --git a/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts index 3ea4173c67..68c0133d76 100644 --- a/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts +++ b/packages/runtime-host/src/__tests__/hosted-execution-tool-profile.test.ts @@ -1,45 +1,74 @@ import assert from 'node:assert/strict'; import test from 'node:test'; +import type { MakaTool } from '@maka/runtime/tool-runtime'; +import { z } from 'zod'; import { decodeHostedExecutionStartInput } from '../protocol/index.js'; -import { HostedExecutionToolProfileRegistry } from '../server/hosted-execution-tool-profile.js'; +import { + hostedExecutionRunProfile, + projectHostedExecutionTools, +} from '../server/hosted-execution-tool-profile.js'; -test('hosted execution tool profiles are explicit protocol inputs', () => { +test('hosted execution tool profiles are durable Session creation inputs', () => { const decoded = decodeHostedExecutionStartInput({ executionId: '00000000-0000-4000-8000-000000000001', session: { workspace: { kind: 'host_path', path: '/workspace' }, modelTarget: { kind: 'explicit', connectionSlug: 'provider', model: 'model' }, + toolProfile: 'headless-coding-v1', }, content: { text: 'solve' }, - toolProfile: 'headless-coding-v1', }); - assert.equal(decoded.toolProfile, 'headless-coding-v1'); + assert.equal(decoded.session.toolProfile, 'headless-coding-v1'); assert.throws( () => decodeHostedExecutionStartInput({ ...decoded, - toolProfile: 'unknown-profile', + session: { ...decoded.session, toolProfile: 'unknown-profile' }, }), - /Invalid Hosted execution tool profile/u, + /Invalid Session tool profile/u, ); }); -test('hosted execution tool profiles remain process-local and bounded to one execution', async () => { - const registry = new HostedExecutionToolProfileRegistry(); - assert.equal(registry.toolNamesFor('execution'), undefined); - assert.equal(registry.allowsMemoryExtractionFor('execution'), true); - await registry.run('execution', 'headless-coding-v1', async () => { - assert.deepEqual(registry.toolNamesFor('execution'), [ - 'Bash', - 'Read', - 'Write', - 'Edit', - 'Glob', - 'Grep', - 'apply_patch', - ]); - assert.equal(registry.allowsMemoryExtractionFor('execution'), false); - }); - assert.equal(registry.toolNamesFor('execution'), undefined); - assert.equal(registry.allowsMemoryExtractionFor('execution'), true); +test('the headless coding profile freezes prompt, tools, memory, and foreground Bash', async () => { + const profile = hostedExecutionRunProfile('headless-coding-v1'); + assert.ok(profile); + assert.deepEqual(profile.toolNames, [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'Glob', + 'Grep', + 'apply_patch', + ]); + assert.equal(profile.memoryExtraction, false); + assert.equal( + profile.systemPrompt, + [ + 'Complete the task by acting with the available tools, not by narrating.', + 'Prefer Read, Glob, and Grep for inspection, Edit and Write for file changes, and Bash for shell commands and tests.', + 'Verify the result when practical.', + 'Stop when the task is complete.', + ].join('\n'), + ); + + const original: MakaTool = { + name: 'Bash', + description: 'Product Bash', + parameters: z.object({ + command: z.string(), + run_in_background: z.boolean().optional(), + pty: z.boolean().optional(), + }), + impl: async () => 'ok', + }; + const [bash] = projectHostedExecutionTools([original], 'headless-coding-v1'); + assert.ok(bash); + const schema = bash.parameters as z.ZodType; + assert.equal((await schema.safeParseAsync({ command: 'true' })).success, true); + assert.equal( + (await schema.safeParseAsync({ command: 'true', run_in_background: true })).success, + false, + ); + assert.equal((await schema.safeParseAsync({ command: 'true', pty: true })).success, false); }); diff --git a/packages/runtime-host/src/protocol/hosted-execution.ts b/packages/runtime-host/src/protocol/hosted-execution.ts index c486f47442..79cdca59dd 100644 --- a/packages/runtime-host/src/protocol/hosted-execution.ts +++ b/packages/runtime-host/src/protocol/hosted-execution.ts @@ -11,9 +11,6 @@ import { defineOperation } from './operation-spec.js'; import { decodeSessionCreateInput, type SessionCreateInput } from './session-catalog.js'; import { decodeMessageContent } from './turn.js'; -export const HOSTED_EXECUTION_TOOL_PROFILES = ['headless-coding-v1'] as const; -export type HostedExecutionToolProfile = (typeof HOSTED_EXECUTION_TOOL_PROFILES)[number]; - const ERRORS = [ 'host_not_ready', 'host_draining', @@ -28,7 +25,6 @@ export interface HostedExecutionStartInput { readonly session: Omit; readonly content: MessageContent; readonly maxSteps?: number; - readonly toolProfile?: HostedExecutionToolProfile; } export interface HostedExecutionReferenceInput { @@ -99,7 +95,7 @@ export function decodeHostedExecutionStartInput(value: unknown): HostedExecution value, 'Hosted execution start input', ['executionId', 'session', 'content'], - ['maxSteps', 'toolProfile'], + ['maxSteps'], ); const executionId = requireEntityId(input.executionId, 'executionId'); const { sessionId: _sessionId, ...session } = decodeSessionCreateInput({ @@ -113,22 +109,9 @@ export function decodeHostedExecutionStartInput(value: unknown): HostedExecution ...(input.maxSteps === undefined ? {} : { maxSteps: requirePositiveCount(input.maxSteps, 'maxSteps') }), - ...(input.toolProfile === undefined - ? {} - : { toolProfile: decodeHostedExecutionToolProfile(input.toolProfile) }), }; } -function decodeHostedExecutionToolProfile(value: unknown): HostedExecutionToolProfile { - if ( - typeof value !== 'string' || - !(HOSTED_EXECUTION_TOOL_PROFILES as readonly string[]).includes(value) - ) { - throw invalidProtocolFrame('Invalid Hosted execution tool profile'); - } - return value as HostedExecutionToolProfile; -} - export function decodeHostedExecutionReferenceInput(value: unknown): HostedExecutionReferenceInput { const input = requireExactRecord(value, 'Hosted execution reference', ['executionId']); return { executionId: requireEntityId(input.executionId, 'executionId') }; diff --git a/packages/runtime-host/src/protocol/session-catalog.ts b/packages/runtime-host/src/protocol/session-catalog.ts index d760150a6d..39ff6cfc55 100644 --- a/packages/runtime-host/src/protocol/session-catalog.ts +++ b/packages/runtime-host/src/protocol/session-catalog.ts @@ -5,9 +5,11 @@ import { isSessionStartMode, type SessionStartMode } from '@maka/core/explore-ag import { isSessionBlockedReason, isSessionStatus, + isSessionToolProfile, type SessionBlockedReason, type SessionStatus, type SessionSubagentProjection, + type SessionToolProfile, } from '@maka/core/session'; import { isThinkingLevel, type ThinkingLevel } from '@maka/core/model-thinking'; import type { ExecutionBoundarySummary } from '@maka/core/sandbox-boundary'; @@ -136,6 +138,7 @@ export interface SessionCreateInput { readonly labels?: readonly string[]; readonly modelTarget: SessionModelTarget; readonly thinkingLevel?: ThinkingLevel; + readonly toolProfile?: SessionToolProfile; readonly permissionMode?: PermissionMode; readonly collaborationMode?: CollaborationMode; readonly orchestrationMode?: OrchestrationMode; @@ -426,6 +429,7 @@ export function decodeSessionCreateInput(value: unknown): SessionCreateInput { 'name', 'labels', 'thinkingLevel', + 'toolProfile', 'permissionMode', 'collaborationMode', 'orchestrationMode', @@ -441,6 +445,9 @@ export function decodeSessionCreateInput(value: unknown): SessionCreateInput { ...(Object.hasOwn(input, 'thinkingLevel') ? { thinkingLevel: thinkingLevel(input.thinkingLevel) } : {}), + ...(Object.hasOwn(input, 'toolProfile') + ? { toolProfile: sessionToolProfile(input.toolProfile) } + : {}), ...(Object.hasOwn(input, 'permissionMode') ? { permissionMode: permissionMode(input.permissionMode) } : {}), @@ -453,6 +460,11 @@ export function decodeSessionCreateInput(value: unknown): SessionCreateInput { }; } +function sessionToolProfile(value: unknown): SessionToolProfile { + if (!isSessionToolProfile(value)) throw invalidProtocolFrame('Invalid Session tool profile'); + return value; +} + function sessionStartMode(value: unknown): SessionStartMode { if (!isSessionStartMode(value)) throw invalidProtocolFrame('Invalid Session start mode'); return value; diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 4a15f7d2fa..4b1c4a7702 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -128,7 +128,7 @@ import { import { HostInteractionCoordinator } from './interaction-coordinator.js'; import { HostInteractiveTurnCoordinator } from './interactive-turn-coordinator.js'; import { ensureBootstrapRuntimePolicy } from './bootstrap-runtime-policy.js'; -import { HostedExecutionToolProfileRegistry } from './hosted-execution-tool-profile.js'; +import { hostedExecutionRunProfile } from './hosted-execution-tool-profile.js'; import { HostMemoryCoordinator } from './memory-coordinator.js'; import { HostMemoryExtractionCoordinator } from './memory-extraction-coordinator.js'; import { MemoryExtractionSessionLane } from './memory-extraction-session-lane.js'; @@ -278,7 +278,6 @@ export async function createExecutionRuntimeHostComposition( }); await stores.messageReceiptStore.beginHostEpoch(context.hostEpoch); const backends = new BackendRegistry(); - const hostedExecutionToolProfiles = new HostedExecutionToolProfileRegistry(); backends.register('fake', (backendContext) => new FakeBackend(backendContext)); const runtimePolicyActivation = new RuntimePolicyActivationGate(); const runtimePolicy = new HostRuntimePolicyCoordinator( @@ -645,12 +644,11 @@ export async function createExecutionRuntimeHostComposition( parentAgentTools: childAgentTools.parentTools, childTools: childAgentTools.childTools, worktreePatchWriteBackAvailable: true, - resolveBoundToolNames: (sessionId) => - hostedExecutionToolProfiles.toolNamesFor(sessionId), }), - ...(hostedExecutionToolProfiles.allowsMemoryExtractionFor(backendContext.sessionId) - ? { memoryExtraction } - : {}), + ...(hostedExecutionRunProfile(backendContext.header.toolProfile)?.memoryExtraction === + false + ? {} + : { memoryExtraction }), artifacts: openedArtifactStore, executionArtifacts, usage: openedUsageStores, @@ -698,11 +696,18 @@ export async function createExecutionRuntimeHostComposition( openedPlanStore.readState(sessionId), runtimePolicyStores.runtimePolicy.getSnapshot(), ]); + const runProfile = hostedExecutionRunProfile(header.toolProfile); return createInteractiveRunComposer({ runtimePolicy: runtimePolicySnapshot, skills, memory: requireMemory(memory), taskLedger, + ...(runProfile + ? { + boundToolNames: runProfile.toolNames, + toolProfile: header.toolProfile, + } + : {}), ...(capabilitySnapshot ? { clientCapabilities: capabilitySnapshot } : {}), builtinTools, hostTools: [...hostTools, ...graphTools], @@ -1245,10 +1250,7 @@ export async function createExecutionRuntimeHostComposition( }, }); const hostedExecutions = new HostHostedExecutionCoordinator( - (input, signal) => - hostedExecutionToolProfiles.run(input.executionId, input.toolProfile, () => - hostedExecutionRunner.run(input, signal), - ), + (input, signal) => hostedExecutionRunner.run(input, signal), context.requestDrain, ); let recoverySessions: Awaited> = []; diff --git a/packages/runtime-host/src/server/hosted-execution-tool-profile.ts b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts index 3618663b67..221ad295dd 100644 --- a/packages/runtime-host/src/server/hosted-execution-tool-profile.ts +++ b/packages/runtime-host/src/server/hosted-execution-tool-profile.ts @@ -1,4 +1,6 @@ -import type { HostedExecutionToolProfile } from '../protocol/index.js'; +import type { SessionToolProfile } from '@maka/core/session'; +import type { MakaTool } from '@maka/runtime/tool-runtime'; +import { z } from 'zod'; const HEADLESS_CODING_V1_TOOL_NAMES = [ 'Bash', @@ -10,55 +12,61 @@ const HEADLESS_CODING_V1_TOOL_NAMES = [ 'apply_patch', ] as const; -interface HostedExecutionToolProfilePolicy { +const HEADLESS_CODING_V1_SYSTEM_PROMPT = [ + 'Complete the task by acting with the available tools, not by narrating.', + 'Prefer Read, Glob, and Grep for inspection, Edit and Write for file changes, and Bash for shell commands and tests.', + 'Verify the result when practical.', + 'Stop when the task is complete.', +].join('\n'); + +const HEADLESS_CODING_V1_BASH_DESCRIPTION = + 'Run a foreground shell command in the session cwd. Use Bash for inspection, builds, tests, and task-local generation. Background execution and PTY sessions are unavailable in this profile.'; + +const HEADLESS_CODING_V1_BASH_PARAMETERS = z + .object({ + command: z.string().describe('The shell command to execute'), + timeout_ms: z.number().int().positive().max(600_000).optional(), + }) + .strict(); + +export interface HostedExecutionRunProfile { readonly toolNames: readonly string[]; + readonly systemPrompt: string; readonly memoryExtraction: boolean; } -function hostedExecutionToolProfilePolicy( - profile: HostedExecutionToolProfile, -): HostedExecutionToolProfilePolicy { +export function hostedExecutionRunProfile( + profile: SessionToolProfile | undefined, +): HostedExecutionRunProfile | undefined { + if (profile === undefined) return undefined; if (profile === 'headless-coding-v1') { return { toolNames: HEADLESS_CODING_V1_TOOL_NAMES, + systemPrompt: HEADLESS_CODING_V1_SYSTEM_PROMPT, memoryExtraction: false, }; } profile satisfies never; - throw new Error('Unknown Hosted execution tool profile'); + throw new Error('Unknown Session tool profile'); } -export function hostedExecutionToolNames(profile: HostedExecutionToolProfile): readonly string[] { - return hostedExecutionToolProfilePolicy(profile).toolNames; +export function hostedExecutionToolNames(profile: SessionToolProfile): readonly string[] { + return hostedExecutionRunProfile(profile)!.toolNames; } -export class HostedExecutionToolProfileRegistry { - readonly #profiles = new Map(); - - async run( - executionId: string, - profile: HostedExecutionToolProfile | undefined, - operation: () => Promise, - ): Promise { - if (!profile) return await operation(); - if (this.#profiles.has(executionId)) { - throw new Error('Hosted execution tool profile is already registered'); - } - this.#profiles.set(executionId, profile); - try { - return await operation(); - } finally { - this.#profiles.delete(executionId); - } - } - - toolNamesFor(executionId: string): readonly string[] | undefined { - const profile = this.#profiles.get(executionId); - return profile ? hostedExecutionToolNames(profile) : undefined; - } - - allowsMemoryExtractionFor(executionId: string): boolean { - const profile = this.#profiles.get(executionId); - return profile ? hostedExecutionToolProfilePolicy(profile).memoryExtraction : true; - } +export function projectHostedExecutionTools( + tools: readonly MakaTool[], + profile: SessionToolProfile | undefined, +): readonly MakaTool[] { + if (profile === undefined) return tools; + hostedExecutionRunProfile(profile); + return tools.map((tool) => + tool.name === 'Bash' + ? { + ...tool, + description: HEADLESS_CODING_V1_BASH_DESCRIPTION, + parameters: HEADLESS_CODING_V1_BASH_PARAMETERS, + } + : tool, + ); } diff --git a/packages/runtime-host/src/server/interactive-run-composer.ts b/packages/runtime-host/src/server/interactive-run-composer.ts index 936e67b732..bba8b23e72 100644 --- a/packages/runtime-host/src/server/interactive-run-composer.ts +++ b/packages/runtime-host/src/server/interactive-run-composer.ts @@ -10,6 +10,7 @@ import { import { activePlanExecution, type PlanSessionState, type PlanStore } from '@maka/core/plan'; import type { PermissionMode } from '@maka/core/permission'; import type { RuntimePolicySnapshot } from '@maka/core/runtime-policy'; +import type { SessionToolProfile } from '@maka/core/session'; import { filterModelVisibleTaskLedgerTasks, renderTaskLedgerPromptText, @@ -69,6 +70,10 @@ import type { import type { HostMemoryCoordinator } from './memory-coordinator.js'; import type { HostSkillCatalogCoordinator } from './skill-catalog-coordinator.js'; import type { CanonicalSkillInventorySnapshot } from './skill-catalog-repository.js'; +import { + hostedExecutionRunProfile, + projectHostedExecutionTools, +} from './hosted-execution-tool-profile.js'; const INTERACTIVE_RUN_COMPOSER_ID = 'maka.interactive'; const INTERACTIVE_RUN_COMPOSER_REVISION = '1'; @@ -87,6 +92,7 @@ export interface InteractiveRunComposerInput { readonly sideConversation?: boolean; readonly boundTools?: readonly MakaTool[]; readonly boundToolNames?: readonly string[]; + readonly toolProfile?: SessionToolProfile; readonly skillBudget?: SkillCatalogBudgetOptions; readonly platform?: NodeJS.Platform; readonly shell?: string; @@ -135,9 +141,10 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) const routedCandidateTools = input.deepResearch ? unscopedCandidateTools.filter(isDeepResearchToolAllowed) : unscopedCandidateTools; - const candidateTools = input.boundToolNames + const boundCandidateTools = input.boundToolNames ? bindToolsByName(routedCandidateTools, input.boundToolNames) : routedCandidateTools; + const candidateTools = projectHostedExecutionTools(boundCandidateTools, input.toolProfile); const activeExecution = input.plan ? activePlanExecution(input.plan.state) : undefined; const selectedTools = input.plan ? selectCollaborationTools({ @@ -169,8 +176,17 @@ export function createInteractiveRunComposer(input: InteractiveRunComposerInput) ), ); const childInstruction = input.childInstruction?.trim(); + const runProfile = hostedExecutionRunProfile(input.toolProfile); const resolvedSystemPrompts = new Map>(); const resolveSystemPrompt = (context: HostModelPromptContext): Promise => { + if (runProfile) { + return Promise.resolve( + Object.freeze({ + text: runProfile.systemPrompt, + sourceRevisions: [], + }), + ); + } const key = `${context.sessionId}\u0000${context.turnId}`; const cached = resolvedSystemPrompts.get(key); if (cached) return cached; @@ -304,7 +320,6 @@ export interface InteractiveRunComposerFactoryInput readonly worktreePatchWriteBackAvailable?: boolean; readonly planStore?: PlanStore; readonly deepResearchTools?: readonly MakaTool[]; - readonly resolveBoundToolNames?: (sessionId: string) => readonly string[] | undefined; } export function createInteractiveRunComposerFactory( @@ -369,8 +384,12 @@ export function createInteractiveRunComposerFactory( ? { sideConversation: true } : {}), ...(boundTools ? { boundTools } : {}), - ...(!boundTools && input.resolveBoundToolNames - ? { boundToolNames: input.resolveBoundToolNames(backendContext.sessionId) } + ...(!boundTools && backendContext.header.toolProfile + ? { + boundToolNames: hostedExecutionRunProfile(backendContext.header.toolProfile)! + .toolNames, + toolProfile: backendContext.header.toolProfile, + } : {}), ...(clientCapabilities ? { clientCapabilities } : {}), ...(input.builtinTools ? { builtinTools: input.builtinTools } : {}), diff --git a/packages/runtime-host/src/server/session-catalog-coordinator.ts b/packages/runtime-host/src/server/session-catalog-coordinator.ts index 52b3c18724..60634e5778 100644 --- a/packages/runtime-host/src/server/session-catalog-coordinator.ts +++ b/packages/runtime-host/src/server/session-catalog-coordinator.ts @@ -270,6 +270,7 @@ export class HostSessionCatalogCoordinator { llmConnectionSlug: model.connectionSlug, model: model.model, ...(input.thinkingLevel === undefined ? {} : { thinkingLevel: input.thinkingLevel }), + ...(input.toolProfile === undefined ? {} : { toolProfile: input.toolProfile }), permissionMode: prepared.permissionMode ?? policy.policy.chatDefaults.permissionMode, collaborationMode: input.collaborationMode ?? 'agent', orchestrationMode: input.orchestrationMode ?? 'default', @@ -738,7 +739,7 @@ function createRequestFingerprint( prepared: PreparedSessionCreate, ): string { const identity = [ - 'session.create.v3', + 'session.create.v4', input.sessionId, input.workspace.kind === 'project' ? ['project', input.workspace.projectId] @@ -749,6 +750,7 @@ function createRequestFingerprint( ? ['default'] : ['explicit', input.modelTarget.connectionSlug, input.modelTarget.model], input.thinkingLevel ?? null, + input.toolProfile ?? null, prepared.permissionMode ?? ['runtime_default'], input.collaborationMode ?? 'agent', input.orchestrationMode ?? 'default', diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index c1e2b198e9..71685c66b7 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -2184,6 +2184,7 @@ function fullHeader(overrides: Partial = {}): SessionHeader { llmConnectionSlug: 'openai', connectionLocked: true, model: 'gpt-5', + toolProfile: 'headless-coding-v1', thinkingLevel: 'high', permissionMode: 'ask', collaborationMode: 'agent', diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index aa8f8c0eeb..fbd4e00a7b 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -48,13 +48,14 @@ import type { import type { CreateSessionInput, SessionListFilter } from '@maka/core/runtime-inputs'; -import type { - SessionHeader, - SessionConversationCopy, - SessionSummary, - StoredMessage, - TurnRecord, - UserMessage, +import { + isSessionToolProfile, + type SessionHeader, + type SessionConversationCopy, + type SessionSummary, + type StoredMessage, + type TurnRecord, + type UserMessage, } from '@maka/core/session'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -937,6 +938,7 @@ function buildSessionHeader( llmConnectionSlug: input.llmConnectionSlug, connectionLocked: false, model: input.model ?? 'default', + ...(input.toolProfile !== undefined ? { toolProfile: input.toolProfile } : {}), permissionMode: input.permissionMode, collaborationMode: input.collaborationMode ?? 'agent', orchestrationMode: input.orchestrationMode ?? 'default', @@ -989,6 +991,7 @@ export function normalizeSessionHeader( typeof header.llmConnectionSlug === 'string' && typeof header.connectionLocked === 'boolean' && typeof header.model === 'string' && + (header.toolProfile === undefined || isSessionToolProfile(header.toolProfile)) && isPermissionMode(header.permissionMode) && isCollaborationMode(header.collaborationMode) && isOrchestrationMode(header.orchestrationMode) && From f7293fd063be0ff5956f4362f515c1638fc41461 Mon Sep 17 00:00:00 2001 From: Builder Date: Thu, 13 Aug 2026 09:55:03 +0000 Subject: [PATCH 13/13] fix(eval): address durable profile review findings --- packages/eval/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eval/README.md b/packages/eval/README.md index a00822c60f..da90e794bb 100644 --- a/packages/eval/README.md +++ b/packages/eval/README.md @@ -53,6 +53,6 @@ surfaces, not a complete defense against a deliberately invented lookup channel; namespace still forces all subject traffic through the audited proxy. Collected Maka runtime files and egress audit logs are represented in attempt artifacts with byte counts and SHA-256 digests. The local image tag remains a machine deployment identity rather than a registry digest; digest -pinning is tracked separately. +pinning is tracked in issue #2953. The experiment directory contains the frozen `experiment.json` and append-only attempt records. There is no second mutable results file. A leftover `.writer.lock` means the previous writer did not complete; remove it only after proving that no writer process remains.