fix(cloud-agent-next): recover exhausted wrapper cleanup leases

services/cloud-agent-next/AGENTS.md

@@ -121,6 +121,8 @@ This pattern blocks API endpoints from running for external contributors who don
 - `SessionMessageState` owns lifecycle/outbox status, terminal effect accounting, and a named immutable `admissionSnapshot` only for post-pending replay validation and recovery; predecessor records normalize into partial `legacyAdmissionConstraints` and never fabricate missing immutable input. Terminal and accepted/sent effects are repairable from pending/alarm replay and events use deterministic uniqueness.
 - Wrapper handoff is currently at-least-once under ambiguous delivery failures: the wrapper forwards prompt/command submissions directly to Kilo and does not query Kilo to suppress or recover duplicate `messageId` submissions. Duplicate prompt/command processing is an accepted edge-case trade-off until Kilo provides an atomic submit-or-return-existing contract.
 - When accepted work has no pending residue and its fenced wrapper runtime/socket is gone, disconnect or liveness expiry first reconciles each accepted message against the DO's stored kilocode events (`getAssistantMessageForUserMessage`): positive terminal evidence (assistant `time.completed` or a terminal assistant error) settles the message as `idle_reconciliation`; anything else terminalizes as wrapper failure without redispatch. There is still no live authoritative Kilo terminal query for redispatch; adding one remains separate lifecycle capability work.
+- Physical wrapper cleanup exhaustion (`WRAPPER_STOP_MAX_ATTEMPTS` reached) is fenced but recoverable: the lease re-observes the sandbox on a slow cadence (`WRAPPER_CLEANUP_EXHAUSTED_RECHECK_MS`) and releases to `none` only after a confirmed `absent` observation, so a wedged sandbox reaped later by the container runtime does not brick the session. Recovery is observation-only (`observeWrappersWithoutWaking`) and never issues another stop: the attempt budget and its rollback fence still hold, and the probe must not wake a stopped container to ask about a process that cannot outlive it. Background rechecks stop after `WRAPPER_CLEANUP_EXHAUSTED_RECHECK_WINDOW_MS` so an unrecoverable exhaustion stops re-arming the DO alarm; explicit sends still force one probe afterwards.
+- A pending-message flush blocked on exhaustion forces one out-of-cadence recheck (`recoverExhaustedDeliveryBlock` → `recheckExhaustedCleanup`) because a user is actively waiting, then retries on the `WRAPPER_CLEANUP_EXHAUSTED` budget before failing closed. The retry budget is what keeps the two halves consistent — recovery takes minutes, so terminalizing on the first blocked attempt would discard messages a later probe would have delivered — and failing closed at the end of it is what keeps a message from sitting `queued` with no terminal signal. The flush failure code must stay authoritative: `INTERNAL` is treated as non-authoritative by `recordPendingFlushFailure` and would terminalize the message under whatever earlier cause it carried.
 - Callback delivery retry policy is paired with `wrangler.jsonc`: `CALLBACK_DELIVERY_MAX_ATTEMPTS` includes the initial attempt, and each Cloud Agent Next callback queue consumer must configure `max_retries` for the remaining redeliveries.
 - Queue/drain emits unfenced `MessageDeliveryRequest`; only `AgentRuntime` may allocate/reuse current identity and construct `FencedWrapperDispatchRequest` with complete `WrapperRunFence` for downstream dispatch.
 - Session creation selects an explicit `ProfileResolutionPolicy` at the handler boundary. Implicit repository/default profile resolution is limited to the closed set of approved session origins; omitted, unknown, and non-approved automation origins fail closed unless they supply an explicit profile id.

services/cloud-agent-next/src/agent-sandbox/cloudflare/cloudflare-agent-sandbox.ts

@@ -770,6 +770,12 @@ export class CloudflareAgentSandbox implements AgentSandbox {
     });
   }
 
+  async observeWrappersWithoutWaking(): Promise<WrapperObservation> {
+    const sandbox = await this.getSandbox();
+    if ((await isSandboxContainerRunning(sandbox)) === false) return { status: 'absent' };
+    return this.discoverSessionWrappers();
+  }
+
   private async observeTarget(_target: WrapperStopTarget): Promise<WrapperObservation> {
     // The lease is session-scoped: confirming absence must account for every
     // physical wrapper carrying this logical session marker, including duplicates.

services/cloud-agent-next/src/agent-sandbox/protocol.ts

@@ -131,6 +131,14 @@ export type EnsuredWrapper =
 export type AgentSandbox = {
   ensureWrapper(request: EnsureWrapperRequest): Promise<EnsuredWrapper>;
   discoverSessionWrappers(): Promise<WrapperObservation>;
+  /**
+   * Observe session wrappers without booting a stopped container. A wrapper is
+   * a process and a process cannot outlive its container, so "container not
+   * running" is already proof of absence. Callers that only need to learn
+   * whether a wrapper survives — not to stop one — use this instead of
+   * `discoverSessionWrappers`, whose container fetch wakes the container.
+   */
+  observeWrappersWithoutWaking(): Promise<WrapperObservation>;
   stopWrappers(request: {
     target: WrapperStopTarget;
     attemptId: string;

services/cloud-agent-next/src/persistence/CloudAgentSession.ts

@@ -806,11 +806,28 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
             };
           }
         },
+        observeWrappers: async () => {
+          if (this.physicalWrapperObserver) return this.physicalWrapperObserver();
+          if (this.orchestrator) return { status: 'absent' };
+          const metadata = await this.getStoredMetadata();
+          if (!metadata) {
+            return { status: 'inspection-failed', error: 'Session metadata unavailable' };
+          }
+          if (
+            getSandboxProvider(metadata) === 'cloudflare' &&
+            !this.env.Sandbox &&
+            !this.env.SandboxSmall
+          ) {
+            return { status: 'absent' };
+          }
+          return createAgentSandbox(this.env, metadata).observeWrappersWithoutWaking();
+        },
         recordSharedSandboxFailover: routeKey =>
           this.sharedSandboxFailoverRecorder
             ? this.sharedSandboxFailoverRecorder(routeKey)
             : recordSharedSandboxFailover(this.env.SHARED_SANDBOX_OVERRIDES, routeKey),
         requestAlarmAtOrBefore: deadline => this.scheduleAlarmAtOrBefore(deadline),
+        isSessionDeletionInProgress: () => this.hasDeletionIntent(),
         getSessionIdForLogs: () => this.sessionId,
       });
     }
@@ -859,6 +876,12 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
           const retryAt = nextWrapperCleanupDeadline(lease);
           return retryAt === undefined ? null : { kind: 'retryable', retryAt };
         },
+        // A blocked flush means a user is waiting; let the supervisor force one
+        // out-of-cadence recheck so a reaped sandbox releases the lease
+        // immediately instead of failing the message on the stale fence.
+        recoverExhaustedDeliveryBlock: async () => {
+          await this.getWrapperSupervisor().recheckExhaustedCleanup();
+        },
         deliver: plan => this.executeDirectly(plan),
         isDeliveryHeld: async () =>
           isWrapperRunFinalizing(await getWrapperRuntimeState(this.ctx.storage)),

services/cloud-agent-next/src/persistence/schemas.test.ts

@@ -4,6 +4,7 @@ import {
   ImagesSchema,
   MCPServerConfigSchema,
   MetadataSchema,
+  modelIdSchema,
   RuntimeAgentSchema,
   RuntimeSkillSchema,
   RuntimeSkillsSchema,
@@ -999,6 +1000,30 @@ describe('MetadataSchema with runtimeSkills', () => {
   });
 });
 
+describe('modelIdSchema', () => {
+  it('accepts standard provider/model IDs', () => {
+    expect(modelIdSchema.parse('anthropic/claude-sonnet-4-20250514')).toBe(
+      'anthropic/claude-sonnet-4-20250514'
+    );
+    expect(modelIdSchema.parse('inclusionai/ling-3.0-flash:free')).toBe(
+      'inclusionai/ling-3.0-flash:free'
+    );
+  });
+
+  it('accepts tilde-prefixed latest aliases', () => {
+    expect(modelIdSchema.parse('~x-ai/grok-latest')).toBe('~x-ai/grok-latest');
+    expect(modelIdSchema.parse('~anthropic/claude-sonnet-latest')).toBe(
+      '~anthropic/claude-sonnet-latest'
+    );
+  });
+
+  it('rejects whitespace and other unsafe characters', () => {
+    expect(modelIdSchema.safeParse('x ai/grok').success).toBe(false);
+    expect(modelIdSchema.safeParse('x;ai/grok').success).toBe(false);
+    expect(modelIdSchema.safeParse('').success).toBe(false);
+  });
+});
+
 describe('RuntimeAgentSchema', () => {
   it('accepts a well-formed custom slug', () => {
     const agent = { slug: 'reviewer', name: 'Reviewer', config: {} };

services/cloud-agent-next/src/persistence/schemas.ts

@@ -175,8 +175,8 @@ export const modelIdSchema = z
   .min(1, 'Model ID cannot be empty')
   .max(255, 'Model ID too long')
   .regex(
-    /^[a-zA-Z0-9._\-/:]+$/,
-    'Model ID can only contain alphanumeric characters, dots, dashes, underscores, slashes, and colons'
+    /^[a-zA-Z0-9._\-/:~]+$/,
+    'Model ID can only contain alphanumeric characters, dots, dashes, underscores, slashes, colons, and tildes'
   );
 
 /**

services/cloud-agent-next/src/session/pending-messages.ts

@@ -29,6 +29,12 @@ const WORKSPACE_CAPACITY_RETRY_DELAYS_MS = [10_000, 30_000, 60_000] as const;
 // minute — give it a short backed-off budget instead of one generic
 // redelivery.
 const GIT_RATE_LIMIT_RETRY_DELAYS_MS = [15_000, 45_000] as const;
+// Wrapper cleanup exhaustion fences delivery, but it is recoverable: the lease
+// releases once the wedged wrapper is observably gone, and every flush attempt
+// forces one observation. Give the message a few spaced attempts so a container
+// reaped moments after exhaustion still delivers, then fail closed rather than
+// leaving it queued for the whole background recheck window.
+const CLEANUP_EXHAUSTED_RETRY_DELAYS_MS = [30_000, 60_000, 120_000] as const;
 // Other pending delivery failures currently get one redelivery after the initial failed attempt.
 const WARM_FOLLOWUP_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const;
 const COLD_INIT_RETRY_DELAYS_MS = [PENDING_FLUSH_RETRY_BASE_DELAY_MS] as const;
@@ -86,6 +92,7 @@ const PendingFlushFailureCodeSchema = z.enum([
   'KILO_SERVER_FAILED',
   'WRAPPER_START_FAILED',
   'WRAPPER_FINALIZING',
+  'WRAPPER_CLEANUP_EXHAUSTED',
   'SANDBOX_CAPABILITY_UNAVAILABLE',
   'NOT_FOUND',
   'BAD_REQUEST',
@@ -498,7 +505,8 @@ export function shouldSkipPendingFlush(message: PendingSessionMessage, now: numb
 /**
  * Reset-eligible modes each start a fresh retry budget on entry (sandbox-connect
  * has a short reconnect budget; sandbox-capacity and git-rate-limit each have a
- * longer backed-off budget for transient, self-clearing conditions).
+ * longer backed-off budget for transient, self-clearing conditions; cleanup-
+ * exhausted has its own recovery budget).
  * Alternating between them must NOT keep resetting, so callers only reset when
  * entering one of these from a non-reset-eligible state.
  */
@@ -508,6 +516,7 @@ function isResetEligibleFailure(
 ): boolean {
   return (
     code === 'SANDBOX_CONNECT_FAILED' ||
+    code === 'WRAPPER_CLEANUP_EXHAUSTED' ||
     (code === 'WORKSPACE_SETUP_FAILED' &&
       (subtype === 'sandbox_storage_full' || subtype === 'git_rate_limited'))
   );
@@ -523,6 +532,7 @@ export async function recordPendingFlushFailure(
     code?:
       | RetryableResultCode
       | PermanentDeliveryResultCode
+      | 'WRAPPER_CLEANUP_EXHAUSTED'
       | 'NOT_FOUND'
       | 'BAD_REQUEST'
       | 'INTERNAL'
@@ -564,11 +574,11 @@ export async function recordPendingFlushFailure(
       ? options.safeFailureMessage
       : undefined;
   // Reset the attempt counter only when a message ENTERS a reset-eligible
-  // transient mode (sandbox-connect, sandbox-capacity, or git-rate-limit) from a
-  // state that is not itself reset-eligible, so each fresh sequence gets its
-  // full backoff budget. When failures alternate between reset-eligible modes
-  // the counter is NOT reset, so attempts accumulate and the message still
-  // exhausts instead of flapping between modes forever.
+  // transient mode (sandbox-connect, sandbox-capacity, git-rate-limit, or
+  // cleanup-exhausted) from a state that is not itself reset-eligible, so each
+  // fresh sequence gets its full backoff budget. When failures alternate between
+  // reset-eligible modes the counter is NOT reset, so attempts accumulate and
+  // the message still exhausts instead of flapping between modes forever.
   const attempts =
     isResetEligibleFailure(flushFailureCode, failureSubtype) &&
     !isResetEligibleFailure(message.lastFlushFailureCode, message.lastFlushFailureSubtype)
@@ -577,13 +587,15 @@ export async function recordPendingFlushFailure(
   const retryDelays =
     flushFailureCode === 'SANDBOX_CONNECT_FAILED'
       ? SANDBOX_CONNECT_RETRY_DELAYS_MS
-      : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'sandbox_storage_full'
-        ? WORKSPACE_CAPACITY_RETRY_DELAYS_MS
-        : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'git_rate_limited'
-          ? GIT_RATE_LIMIT_RETRY_DELAYS_MS
-          : options.policy === 'cold-init'
-            ? COLD_INIT_RETRY_DELAYS_MS
-            : WARM_FOLLOWUP_RETRY_DELAYS_MS;
+      : flushFailureCode === 'WRAPPER_CLEANUP_EXHAUSTED'
+        ? CLEANUP_EXHAUSTED_RETRY_DELAYS_MS
+        : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'sandbox_storage_full'
+          ? WORKSPACE_CAPACITY_RETRY_DELAYS_MS
+          : flushFailureCode === 'WORKSPACE_SETUP_FAILED' && failureSubtype === 'git_rate_limited'
+            ? GIT_RATE_LIMIT_RETRY_DELAYS_MS
+            : options.policy === 'cold-init'
+              ? COLD_INIT_RETRY_DELAYS_MS
+              : WARM_FOLLOWUP_RETRY_DELAYS_MS;
   const retryable = options.retryable ?? isRetryableFlushCode(flushFailureCode);
   const exhausted = !retryable || attempts > retryDelays.length;
   const retryDelay = retryDelays[attempts - 1];
@@ -615,6 +627,7 @@ function isRetryableFlushCode(
   code:
     | RetryableResultCode
     | PermanentDeliveryResultCode
+    | 'WRAPPER_CLEANUP_EXHAUSTED'
     | 'NOT_FOUND'
     | 'BAD_REQUEST'
     | 'INTERNAL'
@@ -629,7 +642,8 @@ function isRetryableFlushCode(
     code === 'SANDBOX_CONNECT_FAILED' ||
     code === 'WORKSPACE_SETUP_FAILED' ||
     code === 'KILO_SERVER_FAILED' ||
-    code === 'WRAPPER_START_FAILED'
+    code === 'WRAPPER_START_FAILED' ||
+    code === 'WRAPPER_CLEANUP_EXHAUSTED'
   );
 }
 export async function deletePendingSessionMessageByMessageId(