Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1656,6 +1656,76 @@ describe('CloudflareAgentSandbox', () => {
).resolves.toMatchObject({ status: 'still-present' });
});

it('confirms absence for an idle-timeout stop without waking a stopped container', async () => {
const listProcesses = vi.fn().mockResolvedValue([]);
const isContainerRunning = vi.fn().mockResolvedValue(false);
const sandbox = new CloudflareAgentSandbox({} as Env, metadata(), {
resolveSandbox: () => ({ listProcesses, isContainerRunning }) as unknown as SandboxInstance,
});

await expect(
sandbox.stopWrappers({
target: { kind: 'session' },
attemptId: 'attempt_idle',
reason: 'idle-timeout',
})
).resolves.toEqual({ status: 'absent' });
// The whole point: no container fetch, so a sleeping container stays asleep.
expect(listProcesses).not.toHaveBeenCalled();
});

it('still inspects an idle-timeout stop while the container is running', async () => {
const listProcesses = vi.fn().mockResolvedValue([]);
const isContainerRunning = vi.fn().mockResolvedValue(true);
const sandbox = new CloudflareAgentSandbox({} as Env, metadata(), {
resolveSandbox: () => ({ listProcesses, isContainerRunning }) as unknown as SandboxInstance,
});

await expect(
sandbox.stopWrappers({
target: { kind: 'session' },
attemptId: 'attempt_idle_running',
reason: 'idle-timeout',
})
).resolves.toEqual({ status: 'absent' });
expect(listProcesses).toHaveBeenCalled();
});

it('inspects a stopped container for stop reasons other than idle-timeout', async () => {
const listProcesses = vi.fn().mockResolvedValue([]);
const isContainerRunning = vi.fn().mockResolvedValue(false);
const sandbox = new CloudflareAgentSandbox({} as Env, metadata(), {
resolveSandbox: () => ({ listProcesses, isContainerRunning }) as unknown as SandboxInstance,
});

await expect(
sandbox.stopWrappers({
target: { kind: 'session' },
attemptId: 'attempt_delete',
reason: 'session-delete',
})
).resolves.toEqual({ status: 'absent' });
expect(listProcesses).toHaveBeenCalled();
expect(isContainerRunning).not.toHaveBeenCalled();
});

it('falls back to inspection when the sandbox cannot report container state', async () => {
const listProcesses = vi.fn().mockResolvedValue([]);
const sandbox = new CloudflareAgentSandbox({} as Env, metadata(), {
// No isContainerRunning: an unknown state must not be treated as "stopped".
resolveSandbox: () => ({ listProcesses }) as unknown as SandboxInstance,
});

await expect(
sandbox.stopWrappers({
target: { kind: 'session' },
attemptId: 'attempt_unknown',
reason: 'idle-timeout',
})
).resolves.toEqual({ status: 'absent' });
expect(listProcesses).toHaveBeenCalled();
});

it('returns inspection-failed from stop when post-stop inspection cannot prove absence', async () => {
const listProcesses = vi
.fn()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,23 @@ import { TOOL_CGROUP_ENV_KEYS, type ToolCgroupEnv } from '../../shared/tool-cgro
import {
buildSandboxBillingInput,
configureSandboxBillingInput,
isSandboxContainerRunning,
type SandboxBillingInput,
} from '../../container-usage-context.js';

const PREPARE_WORKSPACE_TIMEOUT_MS = 10 * 60 * 1000;
const DEFAULT_STOP_OBSERVATION_DELAYS_MS = [100, 500, 1_000];

/**
* Outcome of a wrapper stop inspection, as reported by the `wrapper_stop_inspection` log.
*
* Extends `WrapperObservation` with `absent-no-container`: absence established from
* container state rather than by inspecting, so the log can distinguish a confirmed-empty
* container from one we booted in order to look. Not a `WrapperObservation` status,
* because nothing was observed.
*/
type StopInspection = WrapperObservation | { status: 'absent-no-container' };

function shellQuote(value: string): string {
return `'${value.replaceAll("'", `'"'"'`)}'`;
}
Expand Down Expand Up @@ -751,10 +762,23 @@ export class CloudflareAgentSandbox implements AgentSandbox {
reason: WrapperStopReason;
}): Promise<StopWrappersResult> {
const sandbox = await this.getSandbox();
const initial = await this.observeTarget(request.target);
// Inspection is a container fetch, so it wakes a sleeping container. An `absent`
// result therefore means we booted a container only to learn nothing was running
// in it — the signal for how much idle container time this path is creating.

// Inspecting is a container fetch, so it boots a sleeping container. A wrapper is a
// process, and a process cannot outlive its container (activity expiry SIGTERMs the
// whole container), so a stopped container cannot be hiding a leaked wrapper.
//
// Scoped to idle-timeout: that sweep already established via DO state that no wrapper
// runtime or pending work remains, and it is the path that was waking cold containers
// for nothing. Every other stop reason keeps inspecting, which preserves the leaked
// wrapper recovery those paths were built for.
const skipsInspection =
request.reason === 'idle-timeout' && (await isSandboxContainerRunning(sandbox)) === false;
const initial: StopInspection = skipsInspection
? { status: 'absent-no-container' }
: await this.observeTarget(request.target);

// Single emission for every outcome. `wrapper_stop_inspection` is how container wake
// behaviour is measured, so the field set is defined once here rather than per branch.
logger
.withTags({
logTag: 'wrapper_stop_inspection',
Expand All @@ -769,6 +793,8 @@ export class CloudflareAgentSandbox implements AgentSandbox {
observedWrapperCount: initial.status === 'present' ? initial.observed.length : 0,
})
.info('Wrapper stop inspection completed');

if (initial.status === 'absent-no-container') return { status: 'absent' };
if (initial.status !== 'present') return initial;

try {
Expand Down
26 changes: 26 additions & 0 deletions services/cloud-agent-next/src/container-usage-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export type SandboxBillingInput = Omit<UsageContext, 'service' | 'instanceId' |
};
export type MeteredSandboxInstance = SandboxInstance & {
configureBilling(input: unknown): Promise<void>;
isContainerRunning(): Promise<boolean>;
};

const sandboxBillingInputEnvelopeSchema = z
Expand Down Expand Up @@ -168,6 +169,31 @@ export async function configureSandboxBilling(
await configureSandboxBillingInput(sandbox, buildSandboxBillingInput(metadata, sandboxId));
}

/**
* Whether the sandbox's container is currently running, read over Durable Object RPC.
*
* This deliberately avoids any container fetch (`exec`, `listProcesses`, …), because
* those boot a sleeping container. Callers use it to answer "is there anything running
* in there?" without paying for a wake-up.
*
* Returns `undefined` when the sandbox does not expose the method, so callers can fall
* back to their existing behaviour rather than treating an unknown state as "stopped".
*/
export async function isSandboxContainerRunning(
sandbox: SandboxInstance
): Promise<boolean | undefined> {
const isContainerRunning = (sandbox as Partial<MeteredSandboxInstance>).isContainerRunning;
if (typeof isContainerRunning !== 'function') return undefined;
try {
return await (sandbox as MeteredSandboxInstance).isContainerRunning();
} catch (error) {
logger
.withFields({ error: error instanceof Error ? error.message : String(error) })
.warn('Container running probe failed');
return undefined;
}
}

export async function configureSandboxBillingInput(
sandbox: SandboxInstance,
input: SandboxBillingInput
Expand Down
11 changes: 11 additions & 0 deletions services/cloud-agent-next/src/container-usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,17 @@ export abstract class MeteredSandbox extends StockSandbox<Env> {
});
}

/**
* Whether this sandbox's container is currently running.
*
* Reads Durable Object state only. Calling this over RPC does not boot a sleeping
* container, unlike any container fetch, so callers can confirm "nothing is running
* in there" without paying for a wake-up.
*/
async isContainerRunning(): Promise<boolean> {
return this.ctx.container?.running === true;
}

async configureBilling(input: unknown): Promise<void> {
const parsed = parseSandboxBillingInput(input);
assertSandboxBillingAllocation(this.sandboxClassName, parsed);
Expand Down