test(smoke): prod post-deploy Playwright smoke (5 flows) + rollback-on-red (#227)

.github/workflows/smoke-prod.yml

@@ -0,0 +1,127 @@
+name: Prod Smoke
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'apps/web/**'
+      - 'packages/shared/**'
+  workflow_dispatch:
+    inputs:
+      base_url:
+        description: Override the prod origin to smoke (defaults to SMOKE_BASE_URL secret)
+        required: false
+        type: string
+
+concurrency:
+  group: prod-smoke
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    name: Post-deploy smoke + rollback on red
+    runs-on: ubuntu-latest
+    env:
+      VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
+      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
+      SMOKE_BASE_URL: ${{ inputs.base_url || secrets.SMOKE_BASE_URL }}
+      SMOKE_TEST_EMAIL: ${{ secrets.SMOKE_TEST_EMAIL }}
+      SMOKE_TEST_CODE: ${{ secrets.SMOKE_TEST_CODE }}
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Validate smoke secrets
+        run: |
+          set -euo pipefail
+          missing=()
+          for name in VERCEL_TOKEN VERCEL_ORG_ID VERCEL_PROJECT_ID SMOKE_BASE_URL SMOKE_TEST_EMAIL SMOKE_TEST_CODE; do
+            if [ -z "${!name:-}" ]; then missing+=("$name"); fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            echo "::error::Missing required secrets: ${missing[*]}" >&2
+            exit 1
+          fi
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: 22
+          cache: npm
+
+      - run: npm ci
+
+      - name: Install Playwright chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Install Vercel CLI
+        run: npm i -g vercel@54
+
+      - name: Wait for Vercel production deployment of this commit
+        run: |
+          set -euo pipefail
+          deadline=$(( $(date +%s) + 900 ))
+          api="https://api.vercel.com/v6/deployments?projectId=${VERCEL_PROJECT_ID}&teamId=${VERCEL_ORG_ID}&target=production&sha=${GITHUB_SHA}&limit=1"
+          while :; do
+            body=$(curl -sS -H "Authorization: Bearer ${VERCEL_TOKEN}" "$api")
+            state=$(echo "$body" | jq -r '.deployments[0].readyState // .deployments[0].state // "NONE"')
+            url=$(echo "$body" | jq -r '.deployments[0].url // empty')
+            echo "deployment state for ${GITHUB_SHA}: ${state} (${url:-no-url})"
+            case "$state" in
+              READY)
+                break
+                ;;
+              ERROR|CANCELED|DELETED|BLOCKED)
+                echo "::error::Production deployment for ${GITHUB_SHA} ended in state ${state}; nothing healthy to smoke." >&2
+                exit 1
+                ;;
+            esac
+            if [ "$(date +%s)" -ge "$deadline" ]; then
+              echo "::error::Timed out waiting for the production deployment of ${GITHUB_SHA} to become READY." >&2
+              exit 1
+            fi
+            sleep 15
+          done
+
+      - name: Capture previous production deployment for rollback
+        id: prev_deploy
+        run: |
+          set -euo pipefail
+          api="https://api.vercel.com/v6/deployments?projectId=${VERCEL_PROJECT_ID}&teamId=${VERCEL_ORG_ID}&target=production&state=READY&limit=20"
+          body=$(curl -sS -H "Authorization: Bearer ${VERCEL_TOKEN}" "$api")
+          prev=$(echo "$body" | jq -r --arg sha "$GITHUB_SHA" '
+            [.deployments[] | select((.meta.githubCommitSha // "") != $sha)]
+            | sort_by(-.created) | .[0].url // empty')
+          if [ -z "$prev" ]; then
+            echo "::warning::No previous production deployment found; rollback will be skipped if smoke fails."
+          else
+            echo "previous deployment to roll back to: https://${prev}"
+          fi
+          echo "previous_url=${prev:+https://$prev}" >> "$GITHUB_OUTPUT"
+
+      - name: Run prod smoke suite
+        id: smoke
+        run: npm --workspace @orbit/web run test:smoke
+
+      - name: Roll back the just-shipped deployment (smoke failed)
+        if: failure() && steps.smoke.outcome == 'failure'
+        env:
+          PREVIOUS_URL: ${{ steps.prev_deploy.outputs.previous_url }}
+        run: |
+          set -euo pipefail
+          if [ -z "${PREVIOUS_URL:-}" ]; then
+            echo "::error::Smoke failed but no previous production deployment was captured — manual rollback required at https://vercel.com." >&2
+            exit 1
+          fi
+          echo "::warning::Smoke failed against prod; rolling back to ${PREVIOUS_URL}."
+          vercel rollback "${PREVIOUS_URL}" --non-interactive --timeout 120s
+          echo "::error::Prod smoke failed; rolled back to ${PREVIOUS_URL}. Production auto-deploys are now PAUSED until someone promotes a healthy deployment (vercel promote). Fix forward, then promote." >&2
+
+      - name: Upload Playwright report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: apps/web/e2e/.report
+          if-no-files-found: ignore
+          retention-days: 14

.gitignore

@@ -35,6 +35,13 @@ Thumbs.db
 # Coverage
 coverage/
 
+# Playwright smoke (apps/web/e2e)
+apps/web/e2e/.auth/
+apps/web/e2e/.report/
+apps/web/e2e/.results/
+/test-results/
+.cache/ms-playwright/
+
 # Temporary files
 .tmp/
 temporary screenshots/

apps/web/app/(app)/upgrade/page.tsx

@@ -721,6 +721,7 @@ function PricingSection({
             <PillButton
               fullWidth
               disabled={!!checkoutLoading}
+              dataTestId="paywall-checkout"
               onClick={() => onCheckout(selectedInterval)}
               leading={checkoutLoading
                 ? <Loader2 size={18} className="animate-spin" aria-hidden="true" />

apps/web/app/(auth)/login/code-step.tsx

@@ -86,6 +86,7 @@ export function CodeStep({
           disabled={isSubmitting || codeDigits.join('').length !== 6}
           busy={isSubmitting}
           leading={isSubmitting ? <Spinner /> : undefined}
+          dataTestId="auth-verify-code"
         >
           {t('auth.verify')}
         </PillButton>

apps/web/app/(auth)/login/email-step.tsx

@@ -53,6 +53,7 @@ export function EmailStep({
           disabled={isSubmitting || !email.trim()}
           busy={isSubmitting}
           leading={isSubmitting ? <Spinner /> : undefined}
+          dataTestId="auth-send-code"
         >
           {t('auth.sendCode')}
         </PillButton>

apps/web/app/(chat)/chat/chat-composer-bar.tsx

@@ -270,6 +270,7 @@ function ChatTextInputRow({
         <textarea
           ref={textareaRef}
           rows={1}
+          data-testid="chat-input"
           disabled={limitLocked}
           placeholder={limitLocked ? t('chat.limitReachedError') : t('chat.placeholder')}
           aria-label={t('chat.placeholder')}
@@ -324,6 +325,7 @@ function ChatTextInputRow({
       <button
         type="button"
         disabled={!canSend}
+        data-testid="chat-send"
         aria-label={t('chat.send')}
         onClick={() => sendMessage()}
         className="appearance-none border-0 cursor-pointer inline-flex items-center justify-center shrink-0 rounded-full bg-[var(--primary)] enabled:hover:bg-[var(--primary-pressed)] enabled:hover:scale-105 enabled:hover:shadow-[var(--primary-glow-hover)] enabled:active:scale-95 transition-[background-color,box-shadow,transform] duration-[var(--dur-fast)] ease-[var(--ease-standard)] disabled:cursor-not-allowed"

apps/web/components/chat/pending-operation-card.tsx

@@ -144,6 +144,7 @@ export function PendingOperationCard({
 
   return (
     <div
+      data-testid="pending-op-card"
       className="rounded-[16px] bg-[var(--bg-field)]"
       style={{
         padding: '14px 16px',
@@ -222,6 +223,7 @@ export function PendingOperationCard({
           <PillButton
             className="flex-1 py-[11px]! text-[14px]!"
             disabled={isLoading}
+            dataTestId="pending-op-confirm"
             onClick={() => {
               void handleStart()
             }}

apps/web/components/habits/create-habit-modal.tsx

@@ -364,6 +364,7 @@ export function CreateHabitModal({
             type="submit"
             className="flex-1"
             disabled={isPending || !formHelpers.form.formState.isValid}
+            dataTestId="habit-create-submit"
             leading={
               isPending ? (
                 <Loader2 className="size-[18px] animate-spin" />

apps/web/components/habits/habit-row.tsx

@@ -169,6 +169,8 @@ export function HabitRow({
         }
       }}
       data-tour={tourTargetId}
+      data-testid="habit-row"
+      data-habit-title={habit.title}
       className={
         `relative flex items-center cursor-pointer shadow-[inset_0_0_0_1px_var(--hairline)] transition-[background-color,transform,box-shadow] duration-[160ms] ease-[var(--ease-standard)] active:scale-[0.99] ${
           selected
@@ -558,6 +560,7 @@ function CheckCircle({ state, tone = 'default', onToggle, disabled, ariaLabel }:
   return (
     <button
       type="button"
+      data-testid="habit-status-toggle"
       onClick={(event) => {
         event.stopPropagation()
         if (disabled) return

apps/web/components/ui/pill-button.tsx

@@ -15,6 +15,7 @@ interface PillButtonProps {
   leading?: ReactNode
   children: ReactNode
   className?: string
+  dataTestId?: string
 }
 
 const variantClasses: Record<PillButtonVariant, string> = {
@@ -38,6 +39,7 @@ export function PillButton({
   leading,
   children,
   className,
+  dataTestId,
 }: Readonly<PillButtonProps>) {
   const glowClasses =
     variant === 'primary' && glow && !disabled
@@ -50,6 +52,7 @@ export function PillButton({
       onClick={onClick}
       disabled={disabled}
       aria-busy={busy || undefined}
+      data-testid={dataTestId}
       className={[
         'inline-flex cursor-pointer items-center justify-center gap-[9px] rounded-full border-0 px-[26px] text-[16px] font-medium transition-[background-color,opacity,box-shadow,transform] duration-[var(--dur-fast)] ease-[var(--ease-standard)] disabled:cursor-not-allowed disabled:opacity-40',
         variantClasses[variant],

apps/web/e2e/astra-create-habit.spec.ts

@@ -0,0 +1,27 @@
+import { test, expect } from '@playwright/test'
+import { listHabitTitles } from './support/api'
+import { smokeLabel } from './support/unique'
+
+test('Astra creates a habit from a chat message', async ({ page }) => {
+  const title = smokeLabel('astra')
+
+  await page.goto('/chat')
+
+  const input = page.getByTestId('chat-input')
+  await expect(input).toBeVisible()
+  await input.fill(`Create a daily habit named exactly "${title}". Do not ask for any other details.`)
+  await page.getByTestId('chat-send').click()
+
+  const confirm = page.getByTestId('pending-op-confirm')
+  const confirmAppeared = await confirm
+    .waitFor({ state: 'visible', timeout: 45_000 })
+    .then(() => true)
+    .catch(() => false)
+  if (confirmAppeared) {
+    await confirm.click()
+  }
+
+  await expect
+    .poll(async () => listHabitTitles(page.request), { timeout: 60_000, intervals: [2_000] })
+    .toContain(title)
+})

apps/web/e2e/auth.spec.ts

@@ -0,0 +1,11 @@
+import { test, expect } from '@playwright/test'
+import { authenticate } from './support/auth'
+
+test.use({ storageState: { cookies: [], origins: [] } })
+
+test('login through the passwordless OTP UI reaches the app', async ({ page }) => {
+  await authenticate(page)
+
+  await expect(page).toHaveURL((url) => !url.pathname.startsWith('/login'))
+  await expect(page.locator('[data-bottom-nav]')).toBeVisible()
+})

apps/web/e2e/create-habit.spec.ts

@@ -0,0 +1,20 @@
+import { test, expect } from '@playwright/test'
+import { smokeLabel } from './support/unique'
+
+test('create a habit from the Today FAB', async ({ page }) => {
+  const title = smokeLabel('create')
+
+  await page.goto('/')
+
+  await page.locator('[data-tour="tour-fab-button"]').click()
+
+  const titleInput = page.locator('#habit-form-title')
+  await expect(titleInput).toBeVisible()
+  await titleInput.fill(title)
+
+  const submit = page.getByTestId('habit-create-submit')
+  await expect(submit).toBeEnabled()
+  await submit.click()
+
+  await expect(page.locator(`[data-habit-title="${title}"]`)).toBeVisible()
+})

apps/web/e2e/global.setup.ts

@@ -0,0 +1,15 @@
+import { test as setup, expect } from '@playwright/test'
+import { authenticate } from './support/auth'
+import { resetSmokeAccount } from './support/api'
+import { STORAGE_STATE_PATH } from './support/env'
+
+setup('authenticate and reset the smoke account', async ({ page }) => {
+  await authenticate(page)
+
+  await resetSmokeAccount(page.request)
+
+  const onboarding = await page.request.put('/api/profile/onboarding')
+  expect(onboarding.ok()).toBeTruthy()
+
+  await page.context().storageState({ path: STORAGE_STATE_PATH })
+})

apps/web/e2e/global.teardown.ts

@@ -0,0 +1,6 @@
+import { test as teardown } from '@playwright/test'
+import { resetSmokeAccount } from './support/api'
+
+teardown('wipe smoke data from prod', async ({ page }) => {
+  await resetSmokeAccount(page.request)
+})

apps/web/e2e/log-habit.spec.ts

@@ -0,0 +1,25 @@
+import { test, expect } from '@playwright/test'
+import { smokeLabel } from './support/unique'
+
+test('log a habit from the Today list', async ({ page }) => {
+  const title = smokeLabel('log')
+
+  await page.goto('/')
+
+  await page.locator('[data-tour="tour-fab-button"]').click()
+  const titleInput = page.locator('#habit-form-title')
+  await expect(titleInput).toBeVisible()
+  await titleInput.fill(title)
+  const submit = page.getByTestId('habit-create-submit')
+  await expect(submit).toBeEnabled()
+  await submit.click()
+
+  const row = page.locator(`[data-habit-title="${title}"]`)
+  await expect(row).toBeVisible()
+
+  const toggle = row.getByTestId('habit-status-toggle')
+  const initialState = await toggle.getAttribute('aria-label')
+  await toggle.click()
+
+  await expect(toggle).not.toHaveAttribute('aria-label', initialState ?? '')
+})

apps/web/e2e/paywall.spec.ts

@@ -0,0 +1,8 @@
+import { test, expect } from '@playwright/test'
+
+test('the upgrade paywall renders its checkout CTA', async ({ page }) => {
+  await page.goto('/upgrade')
+
+  await expect(page.getByRole('radiogroup')).toBeVisible()
+  await expect(page.getByTestId('paywall-checkout')).toBeVisible()
+})

apps/web/e2e/support/api.ts

@@ -0,0 +1,25 @@
+import type { APIRequestContext } from '@playwright/test'
+
+/** BFF helpers used by setup/teardown and by the Astra spec to assert server
+ *  state directly. They reuse the page's session cookie via `page.request`, so
+ *  every call is authenticated as the smoke user. */
+
+export async function resetSmokeAccount(request: APIRequestContext): Promise<void> {
+  const response = await request.post('/api/profile/reset')
+  if (!response.ok()) {
+    throw new Error(`profile reset failed: ${response.status()} ${await response.text()}`)
+  }
+}
+
+interface HabitListItem {
+  title: string
+}
+
+export async function listHabitTitles(request: APIRequestContext): Promise<string[]> {
+  const response = await request.get('/api/habits')
+  if (!response.ok()) {
+    throw new Error(`list habits failed: ${response.status()} ${await response.text()}`)
+  }
+  const body = (await response.json()) as { items?: HabitListItem[] }
+  return (body.items ?? []).map((habit) => habit.title)
+}