Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions packages/status/email-policy.node.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,48 @@ test('composed emails carry component names, status page link, and escape html',
})
expect(opened.subject).toContain('App & API')
})

test('outage emails annotate active relevant Cloudflare incidents', () => {
const providerIncidents = [
{
id: 'inc-r2',
name: 'R2 Availability Issues',
status: 'investigating',
impact: 'minor',
shortlink: 'https://stspg.io/r2',
updatedAt: '2026-08-07T19:00:00.000Z',
affectedComponents: ['R2'],
},
]
const opened = composeStatusEmail({
kind: 'incident_opened',
openIncidents: [openIncident()],
statusPageUrl: 'https://status.heykody.dev',
now: baseNow,
providerIncidents,
})
expect(opened.text).toContain(
'Possibly related Cloudflare incident: R2 Availability Issues (investigating)',
)
expect(opened.html).toContain('Possibly related Cloudflare incident')

const reminder = composeStatusEmail({
kind: 'daily_reminder',
openIncidents: [openIncident()],
statusPageUrl: 'https://status.heykody.dev',
now: baseNow,
providerIncidents,
})
expect(reminder.text).toContain(
'Possibly related Cloudflare incident: R2 Availability Issues (investigating)',
)

const allClear = composeStatusEmail({
kind: 'all_clear',
openIncidents: [],
statusPageUrl: 'https://status.heykody.dev',
now: baseNow,
providerIncidents,
})
expect(allClear.text).not.toContain('Possibly related Cloudflare incident')
})
17 changes: 14 additions & 3 deletions packages/status/email-policy.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { statusComponentName, type StatusComponentId } from './status-types.ts'
import { formatProviderIncidentAnnotation } from './provider-incidents.ts'
import {
statusComponentName,
type ProviderIncident,
type StatusComponentId,
} from './status-types.ts'

/**
* Operator alert policy:
Expand Down Expand Up @@ -85,23 +90,29 @@ export function composeStatusEmail(input: {
openIncidents: Array<OpenIncidentSummary>
statusPageUrl: string
now: number
providerIncidents?: Array<ProviderIncident> | null
}): StatusEmailContent {
const componentNames = input.openIncidents
.map((incident) => statusComponentName(incident.component))
.join(', ')
const incidentLines = describeIncidents(input.openIncidents)
const providerAnnotation =
input.kind === 'all_clear'
? null
: formatProviderIncidentAnnotation(input.providerIncidents ?? [])
const providerBlock = providerAnnotation ? `\n\n${providerAnnotation}` : ''
const footerText = `Status page: ${input.statusPageUrl}`
const footerHtml = `<p><a href="${escapeHtml(input.statusPageUrl)}">Open the status page</a></p>`

switch (input.kind) {
case 'incident_opened': {
const subject = `[kody status] Incident: ${componentNames} down`
const text = `An incident is affecting kody.\n\n${incidentLines}\n\nYou will get at most one reminder per day while this is unresolved, and an all-clear email once everything recovers.\n\n${footerText}`
const text = `An incident is affecting kody.\n\n${incidentLines}${providerBlock}\n\nYou will get at most one reminder per day while this is unresolved, and an all-clear email once everything recovers.\n\n${footerText}`
return { subject, text, html: toHtml(text, footerHtml) }
}
case 'daily_reminder': {
const subject = `[kody status] Reminder: incident still unresolved (${componentNames})`
const text = `An incident affecting kody is still unresolved.\n\n${incidentLines}\n\n${footerText}`
const text = `An incident affecting kody is still unresolved.\n\n${incidentLines}${providerBlock}\n\n${footerText}`
return { subject, text, html: toHtml(text, footerHtml) }
}
case 'all_clear': {
Expand Down
229 changes: 229 additions & 0 deletions packages/status/provider-incidents.node.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
import { expect, test, vi } from 'vitest'
import {
cloudflareStatusPageUrl,
fetchRelevantProviderIncidents,
filterRelevantProviderIncidents,
formatProviderIncidentAnnotation,
parseProviderIncidentCache,
parseProviderIncidentsPayload,
providerIncidentCacheStaleMs,
sanitizeProviderIncidentShortlink,
serializeProviderIncidentCache,
type ProviderIncident,
} from './provider-incidents.ts'

const r2Incident = {
id: 'inc-r2',
name: 'R2 Availability Issues',
status: 'investigating',
impact: 'minor',
shortlink: 'https://stspg.io/r2',
updated_at: '2026-08-07T19:00:00.000Z',
components: [{ id: 'hb7g5sq2zz0h', name: 'R2' }],
incident_updates: [
{
affected_components: [
{
code: 'hb7g5sq2zz0h',
name: 'Cloudflare Sites and Services - R2',
},
],
},
],
}

const streamIncident = {
id: 'inc-stream',
name: 'Stream hiccup',
status: 'identified',
impact: 'major',
shortlink: 'https://stspg.io/stream',
updated_at: '2026-08-07T18:00:00.000Z',
components: [{ id: 'stream-id', name: 'Stream' }],
}

const popIncident = {
id: 'inc-pop',
name: 'PoP degradation',
status: 'monitoring',
impact: 'minor',
shortlink: 'https://stspg.io/pop',
updated_at: '2026-08-07T17:00:00.000Z',
components: [{ id: 'ams', name: 'Amsterdam, Netherlands - (AMS)' }],
}

function providerIncident(
overrides: Partial<ProviderIncident> = {},
): ProviderIncident {
return {
id: 'inc-r2',
name: 'R2 Availability Issues',
status: 'investigating',
impact: 'minor',
shortlink: 'https://stspg.io/r2',
updatedAt: '2026-08-07T19:00:00.000Z',
affectedComponents: ['R2'],
...overrides,
}
}

test('filters Cloudflare incidents to products kody runs on', () => {
const filtered = filterRelevantProviderIncidents([
r2Incident,
streamIncident,
popIncident,
{
id: 'inc-workers',
name: 'Workers elevated errors',
status: 'identified',
impact: 'major',
shortlink: 'https://stspg.io/workers',
updated_at: '2026-08-07T16:00:00.000Z',
components: [],
incident_updates: [
{
affected_components: [
{ name: 'Cloudflare Sites and Services - Workers' },
{ name: 'Cloudflare Sites and Services - Stream' },
],
},
],
},
])
expect(filtered.map((incident) => incident.id)).toEqual([
'inc-r2',
'inc-workers',
])
expect(filtered[0]?.affectedComponents).toEqual(['R2'])
expect(filtered[1]?.affectedComponents).toEqual(['Workers'])
expect(
parseProviderIncidentsPayload({ incidents: [streamIncident] }),
).toEqual([])
})

test('provider incident fetch fails soft on timeout, http errors, and bad JSON', async () => {
const consoleWarn = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
const timedOut = await fetchRelevantProviderIncidents({
timeoutMs: 5,
fetcher: async () =>
await new Promise(() => {
/* never resolves */
}),
})
expect(timedOut.ok).toBe(false)
expect(timedOut.incidents).toEqual([])

const httpError = await fetchRelevantProviderIncidents({
fetcher: async () => new Response('nope', { status: 503 }),
})
expect(httpError).toEqual({
ok: false,
incidents: [],
error: 'HTTP 503',
})

const badJson = await fetchRelevantProviderIncidents({
fetcher: async () =>
new Response('not-json', {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
})
expect(badJson.ok).toBe(false)

const okEmpty = await fetchRelevantProviderIncidents({
fetcher: async () =>
Response.json({
incidents: [streamIncident, popIncident],
}),
})
expect(okEmpty).toEqual({ ok: true, incidents: [] })

const okRelevant = await fetchRelevantProviderIncidents({
fetcher: async () => Response.json({ incidents: [r2Incident] }),
})
expect(okRelevant.ok).toBe(true)
expect(okRelevant.incidents).toHaveLength(1)
expect(okRelevant.incidents[0]?.name).toBe('R2 Availability Issues')
expect(consoleWarn).toHaveBeenCalled()
} finally {
consoleWarn.mockRestore()
}
})

test('provider incident cache parses within TTL and rejects stale or corrupt entries', () => {
const now = Date.UTC(2026, 7, 7, 20, 0, 0)
const fresh = serializeProviderIncidentCache({
fetchedAt: now - 30_000,
incidents: [providerIncident()],
})
expect(parseProviderIncidentCache(fresh, now)?.[0]?.name).toBe(
'R2 Availability Issues',
)
expect(
parseProviderIncidentCache(fresh, now + providerIncidentCacheStaleMs + 1),
).toBeNull()
expect(parseProviderIncidentCache('not-json', now)).toBeNull()
expect(parseProviderIncidentCache(null, now)).toBeNull()

const incomplete = JSON.stringify({
fetchedAt: now - 1_000,
incidents: [
{
id: 'inc-partial',
name: 'Missing fields',
status: 'investigating',
impact: 'minor',
shortlink: 'https://stspg.io/x',
// updatedAt / affectedComponents omitted on purpose
},
],
})
expect(parseProviderIncidentCache(incomplete, now)).toEqual([])

const futureDated = serializeProviderIncidentCache({
fetchedAt: now + 60_000,
incidents: [providerIncident()],
})
expect(parseProviderIncidentCache(futureDated, now)).toBeNull()
})

test('provider shortlinks are restricted to Cloudflare Statuspage HTTPS hosts', () => {
expect(sanitizeProviderIncidentShortlink('https://stspg.io/r2')).toBe(
'https://stspg.io/r2',
)
expect(sanitizeProviderIncidentShortlink('javascript:alert(1)')).toBe(
cloudflareStatusPageUrl,
)
expect(sanitizeProviderIncidentShortlink('https://evil.example/phish')).toBe(
cloudflareStatusPageUrl,
)

const filtered = filterRelevantProviderIncidents([
{
...r2Incident,
shortlink: 'javascript:alert(1)',
},
])
expect(filtered[0]?.shortlink).toBe(cloudflareStatusPageUrl)
})

test('outage email annotation names active Cloudflare incidents', () => {
expect(formatProviderIncidentAnnotation([])).toBeNull()
expect(
formatProviderIncidentAnnotation([
providerIncident(),
providerIncident({
id: 'inc-d1',
name: 'D1 query errors',
status: 'monitoring',
}),
]),
).toBe(
[
'Possibly related Cloudflare incident: R2 Availability Issues (investigating)',
'Possibly related Cloudflare incident: D1 query errors (monitoring)',
].join('\n'),
)
})
Loading
Loading