Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions packages/worker/src/sentry-options.node.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { expect, test } from 'vitest'
import { isUserCodeError, UserCodeError } from './user-code-error.ts'
import {
durableObjectIsolateCpuResetMessage,
durableObjectIsolateMemoryResetMessage,
executorSandboxTimeoutMessage,
filterSentryEvent,
} from './sentry-options.ts'
Expand Down Expand Up @@ -183,4 +185,78 @@ test('filterSentryEvent drops expected platform and caller noise and keeps real
}),
).toBe(platformEvent)
expect(filterSentryEvent(platformEvent)).toBe(platformEvent)

// Bare Cloudflare DO isolate resets (memory / CPU) are transient platform
// limit hits — drop exact strings, including optional `Error:` prefix and
// missing trailing period. Wrapped recovery failures must stay visible.
expect(
filterSentryEvent({
exception: {
values: [{ value: durableObjectIsolateMemoryResetMessage }],
},
}),
).toBeNull()
expect(
filterSentryEvent({
exception: {
values: [
{
value:
"Error: Durable Object's isolate exceeded its memory limit and was reset.",
},
],
},
}),
).toBeNull()
expect(
filterSentryEvent({
exception: {
values: [
{
value: 'Durable Object exceeded its CPU time limit and was reset',
},
],
},
}),
).toBeNull()
expect(
filterSentryEvent({
message: durableObjectIsolateCpuResetMessage,
}),
).toBeNull()

const exhaustedPublishRecoveryInline = {
exception: {
values: [
{
value: `package_publish_external_push could not recover after 3 transient Durable Object reset attempts: ${durableObjectIsolateMemoryResetMessage}`,
},
],
},
}
expect(filterSentryEvent(exhaustedPublishRecoveryInline)).toBe(
exhaustedPublishRecoveryInline,
)

const exhaustedPublishRecoveryChained = {
exception: {
values: [
{
value:
'package_publish_external_push could not recover after 3 transient Durable Object reset attempts',
},
{ value: durableObjectIsolateMemoryResetMessage },
],
},
}
expect(filterSentryEvent(exhaustedPublishRecoveryChained)).toBe(
exhaustedPublishRecoveryChained,
)

const unrelatedDoFailure = {
exception: {
values: [{ value: 'Durable Object was reset during migration' }],
},
}
expect(filterSentryEvent(unrelatedDoFailure)).toBe(unrelatedDoFailure)
})
53 changes: 52 additions & 1 deletion packages/worker/src/sentry-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,62 @@ export function filterExecutorSandboxTimeoutSentryEvent(event: ErrorEvent) {
return null
}

/**
* Exact Cloudflare Durable Object isolate-reset messages. When a DO hits its
* memory or CPU limit, the platform resets the isolate and surfaces this error
* to the RPC caller (for example `job_run_now` → JobManager). The next call
* gets a fresh isolate; app-level retry is unsafe for non-idempotent jobs, and
* moving heavy orchestration off JobManager is an architectural change outside
* a triage fix. Match only the bare platform strings so wrapped failures such
* as exhausted `package_publish_external_push` recovery messages stay visible.
*/
export const durableObjectIsolateMemoryResetMessage =
"Durable Object's isolate exceeded its memory limit and was reset."

export const durableObjectIsolateCpuResetMessage =
'Durable Object exceeded its CPU time limit and was reset.'

function normalizeDurableObjectIsolateResetMessage(message: string) {
const withoutErrorPrefix = message.trim().replace(/^Error:\s*/i, '')
return withoutErrorPrefix.endsWith('.')
? withoutErrorPrefix
: `${withoutErrorPrefix}.`
}

export function isDurableObjectIsolateResetMessage(message: string) {
const normalized = normalizeDurableObjectIsolateResetMessage(message)
return (
normalized === durableObjectIsolateMemoryResetMessage ||
normalized === durableObjectIsolateCpuResetMessage
)
}

export function isDurableObjectIsolateResetSentryEvent(event: ErrorEvent) {
const messages = sentryEventMessages(event).filter(
(message): message is string =>
typeof message === 'string' && message.trim().length > 0,
)
// Drop only when every reported message is a bare reset. A chained cause
// that pairs a recovery/wrapper value with an inner reset must stay visible.
return (
messages.length > 0 &&
messages.every((message) => isDurableObjectIsolateResetMessage(message))
)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

export function filterDurableObjectIsolateResetSentryEvent(event: ErrorEvent) {
if (!isDurableObjectIsolateResetSentryEvent(event)) return event
return null
}

export function filterSentryEvent(event: ErrorEvent, hint?: EventHint) {
// Marker first: primary mechanism for user-authored failures.
if (filterUserCodeErrorSentryEvent(event, hint) === null) return null
if (filterRetryableD1LockSentryEvent(event) === null) return null
// String-match backstops for paths that cannot yet be marked.
if (filterUserModuleBundlerFailureSentryEvent(event) === null) return null
if (filterExecutorSandboxTimeoutSentryEvent(event) === null) return null
if (filterDurableObjectIsolateResetSentryEvent(event) === null) return null
return event
}

Expand Down Expand Up @@ -122,7 +171,9 @@ export function buildSentryOptions(env: Env): CloudflareOptions {
// (`filterUserCodeErrorSentryEvent`). Bundler / sandbox-timeout string
// matches remain as backstops for unmarked paths; see
// filterUserModuleBundlerFailureSentryEvent and
// filterExecutorSandboxTimeoutSentryEvent.
// filterExecutorSandboxTimeoutSentryEvent. Bare Cloudflare Durable
// Object isolate memory/CPU reset strings are dropped the same way —
// see filterDurableObjectIsolateResetSentryEvent.
beforeSend: filterSentryEvent,
}
}
Expand Down
Loading