diff --git a/packages/worker/src/sentry-options.node.test.ts b/packages/worker/src/sentry-options.node.test.ts index 22a61af412..ad39b5b808 100644 --- a/packages/worker/src/sentry-options.node.test.ts +++ b/packages/worker/src/sentry-options.node.test.ts @@ -1,6 +1,8 @@ import { expect, test } from 'vitest' import { isUserCodeError, UserCodeError } from './user-code-error.ts' import { + durableObjectIsolateCpuResetMessage, + durableObjectIsolateMemoryResetMessage, executorSandboxTimeoutMessage, filterSentryEvent, } from './sentry-options.ts' @@ -183,4 +185,78 @@ test('filterSentryEvent drops expected platform and caller noise and keeps real }), ).toBe(platformEvent) expect(filterSentryEvent(platformEvent)).toBe(platformEvent) + + // Bare Cloudflare DO isolate resets (memory / CPU) are transient platform + // limit hits — drop exact strings, including optional `Error:` prefix and + // missing trailing period. Wrapped recovery failures must stay visible. + expect( + filterSentryEvent({ + exception: { + values: [{ value: durableObjectIsolateMemoryResetMessage }], + }, + }), + ).toBeNull() + expect( + filterSentryEvent({ + exception: { + values: [ + { + value: + "Error: Durable Object's isolate exceeded its memory limit and was reset.", + }, + ], + }, + }), + ).toBeNull() + expect( + filterSentryEvent({ + exception: { + values: [ + { + value: 'Durable Object exceeded its CPU time limit and was reset', + }, + ], + }, + }), + ).toBeNull() + expect( + filterSentryEvent({ + message: durableObjectIsolateCpuResetMessage, + }), + ).toBeNull() + + const exhaustedPublishRecoveryInline = { + exception: { + values: [ + { + value: `package_publish_external_push could not recover after 3 transient Durable Object reset attempts: ${durableObjectIsolateMemoryResetMessage}`, + }, + ], + }, + } + expect(filterSentryEvent(exhaustedPublishRecoveryInline)).toBe( + exhaustedPublishRecoveryInline, + ) + + const exhaustedPublishRecoveryChained = { + exception: { + values: [ + { + value: + 'package_publish_external_push could not recover after 3 transient Durable Object reset attempts', + }, + { value: durableObjectIsolateMemoryResetMessage }, + ], + }, + } + expect(filterSentryEvent(exhaustedPublishRecoveryChained)).toBe( + exhaustedPublishRecoveryChained, + ) + + const unrelatedDoFailure = { + exception: { + values: [{ value: 'Durable Object was reset during migration' }], + }, + } + expect(filterSentryEvent(unrelatedDoFailure)).toBe(unrelatedDoFailure) }) diff --git a/packages/worker/src/sentry-options.ts b/packages/worker/src/sentry-options.ts index a48a5de6dc..327a3b6d9b 100644 --- a/packages/worker/src/sentry-options.ts +++ b/packages/worker/src/sentry-options.ts @@ -81,6 +81,54 @@ export function filterExecutorSandboxTimeoutSentryEvent(event: ErrorEvent) { return null } +/** + * Exact Cloudflare Durable Object isolate-reset messages. When a DO hits its + * memory or CPU limit, the platform resets the isolate and surfaces this error + * to the RPC caller (for example `job_run_now` → JobManager). The next call + * gets a fresh isolate; app-level retry is unsafe for non-idempotent jobs, and + * moving heavy orchestration off JobManager is an architectural change outside + * a triage fix. Match only the bare platform strings so wrapped failures such + * as exhausted `package_publish_external_push` recovery messages stay visible. + */ +export const durableObjectIsolateMemoryResetMessage = + "Durable Object's isolate exceeded its memory limit and was reset." + +export const durableObjectIsolateCpuResetMessage = + 'Durable Object exceeded its CPU time limit and was reset.' + +function normalizeDurableObjectIsolateResetMessage(message: string) { + const withoutErrorPrefix = message.trim().replace(/^Error:\s*/i, '') + return withoutErrorPrefix.endsWith('.') + ? withoutErrorPrefix + : `${withoutErrorPrefix}.` +} + +export function isDurableObjectIsolateResetMessage(message: string) { + const normalized = normalizeDurableObjectIsolateResetMessage(message) + return ( + normalized === durableObjectIsolateMemoryResetMessage || + normalized === durableObjectIsolateCpuResetMessage + ) +} + +export function isDurableObjectIsolateResetSentryEvent(event: ErrorEvent) { + const messages = sentryEventMessages(event).filter( + (message): message is string => + typeof message === 'string' && message.trim().length > 0, + ) + // Drop only when every reported message is a bare reset. A chained cause + // that pairs a recovery/wrapper value with an inner reset must stay visible. + return ( + messages.length > 0 && + messages.every((message) => isDurableObjectIsolateResetMessage(message)) + ) +} + +export function filterDurableObjectIsolateResetSentryEvent(event: ErrorEvent) { + if (!isDurableObjectIsolateResetSentryEvent(event)) return event + return null +} + export function filterSentryEvent(event: ErrorEvent, hint?: EventHint) { // Marker first: primary mechanism for user-authored failures. if (filterUserCodeErrorSentryEvent(event, hint) === null) return null @@ -88,6 +136,7 @@ export function filterSentryEvent(event: ErrorEvent, hint?: EventHint) { // String-match backstops for paths that cannot yet be marked. if (filterUserModuleBundlerFailureSentryEvent(event) === null) return null if (filterExecutorSandboxTimeoutSentryEvent(event) === null) return null + if (filterDurableObjectIsolateResetSentryEvent(event) === null) return null return event } @@ -122,7 +171,9 @@ export function buildSentryOptions(env: Env): CloudflareOptions { // (`filterUserCodeErrorSentryEvent`). Bundler / sandbox-timeout string // matches remain as backstops for unmarked paths; see // filterUserModuleBundlerFailureSentryEvent and - // filterExecutorSandboxTimeoutSentryEvent. + // filterExecutorSandboxTimeoutSentryEvent. Bare Cloudflare Durable + // Object isolate memory/CPU reset strings are dropped the same way — + // see filterDurableObjectIsolateResetSentryEvent. beforeSend: filterSentryEvent, } }