Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 39 additions & 51 deletions .github/workflows/payment-sandbox-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@
#
# It is deliberately NOT a PR gate: it needs network to third-party sandboxes
# and is inherently slower/flakier than mocked tests. It runs nightly and on
# demand. Square and SumUp skip when unconfigured, so they can be enabled one at
# a time. The free and Stripe legs must execute; a missing Stripe secret fails.
# demand. Every leg must execute: a missing provider secret fails its leg.
#
# Required repository secrets (add only the providers you use):
# Required repository secrets:
# STRIPE_SECRET_KEY — sk_test_… (Stripe test mode)
# SQUARE_ACCESS_TOKEN — Square sandbox access token
# SQUARE_LOCATION_ID — Square sandbox location id
Expand Down Expand Up @@ -46,18 +45,14 @@ concurrency:
jobs:
e2e:
runs-on: ubuntu-latest
# The harness bounds every step, scenario, and teardown internally, so a
# healthy leg finishes well inside this. The limit stops a stalled runner
# step (for example a hung apt mirror) from holding a leg for hours.
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- target: free
must_execute: true
- target: stripe
must_execute: true
- target: square
must_execute: false
- target: sumup
must_execute: false
target: [free, stripe, square, sumup]
steps:
- name: Checkout repository
uses: actions/checkout@v4
Expand All @@ -76,43 +71,41 @@ jobs:
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
"$RUNNER_TEMP/bin/cloudflared" --version

- name: Resolve Playwright version
id: pw
# The ubuntu-latest image ships Google Chrome, so the usual path needs
# no browser download and no apt install of OS libraries. If a future
# image drops Chrome, fall back to the Playwright-pinned Chromium with
# its OS dependencies, resolved from e2e-payments/deno.json.
- name: Provide a browser
id: browser
run: |
version="$(
deno eval '
const config = JSON.parse(await Deno.readTextFile("e2e-payments/deno.json"));
const spec = config.imports?.playwright;
const version = spec?.match(/^npm:playwright@(.+)$/)?.[1];
if (!version) {
throw new Error(`Expected imports.playwright to be npm:playwright@<version>; got ${spec}`);
}
console.log(version);
'
)"
echo "version=$version" >> "$GITHUB_OUTPUT"

- name: Cache Playwright browsers
id: pw-cache
uses: actions/cache@v4
with:
path: ~/.cache/ms-playwright
key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }}

# On a cache hit the browser binary is already restored, so install only
# the OS-level libraries (apt) Chromium needs — those can't be cached. On
# a miss, download the browser and its OS deps in one go.
- name: Install Playwright Chromium (cache miss)
if: steps.pw-cache.outputs.cache-hit != 'true'
run: deno run -A npm:playwright@${{ steps.pw.outputs.version }} install --with-deps chromium

- name: Install Playwright OS dependencies (cache hit)
if: steps.pw-cache.outputs.cache-hit == 'true'
run: deno run -A npm:playwright@${{ steps.pw.outputs.version }} install-deps chromium
set -euo pipefail
if command -v google-chrome >/dev/null; then
echo "executable=$(command -v google-chrome)" >> "$GITHUB_OUTPUT"
google-chrome --version
else
echo "No system Chrome on this image; installing Playwright Chromium."
version="$(
deno eval '
const config = JSON.parse(await Deno.readTextFile("e2e-payments/deno.json"));
const spec = config.imports?.playwright;
const version = spec?.match(/^npm:playwright@(.+)$/)?.[1];
if (!version) {
throw new Error(`Expected imports.playwright to be npm:playwright@<version>; got ${spec}`);
}
console.log(version);
'
)"
deno run -A npm:playwright@"$version" install --with-deps chromium
echo "executable=" >> "$GITHUB_OUTPUT"
fi

- name: Run ${{ matrix.target }} payment e2e
id: payment-run
env:
# Empty when the fallback installed Playwright's own Chromium — the
# harness treats an empty value as unset and lets Playwright resolve
# its bundled browser.
CHROMIUM_EXECUTABLE: ${{ steps.browser.outputs.executable }}
# Throwaway key for the ephemeral local DB — not a real secret.
DB_ENCRYPTION_KEY: MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=
STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
Expand All @@ -127,14 +120,9 @@ jobs:
- name: Verify payment leg execution
env:
ACTUAL_RESULT: ${{ steps.payment-run.outputs.result }}
MUST_EXECUTE: ${{ matrix.must_execute }}
run: |
if [[ "$ACTUAL_RESULT" != "executed" && "$ACTUAL_RESULT" != "skipped" ]]; then
echo "Payment harness did not report whether the leg executed or skipped." >&2
exit 1
fi
if [[ "$MUST_EXECUTE" == "true" && "$ACTUAL_RESULT" != "executed" ]]; then
echo "The ${{ matrix.target }} payment leg is configured to run but skipped." >&2
if [[ "$ACTUAL_RESULT" != "executed" ]]; then
echo "The ${{ matrix.target }} payment leg did not report that it executed." >&2
exit 1
fi

Expand Down
67 changes: 10 additions & 57 deletions TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -2485,63 +2485,16 @@ scope so intermediate applies never touch tables the migration does not declare.
The empty-rows guard in `clearDormantPaymentTables` is the model for the drop;
keep the loud refusal on non-empty tables.

## Harden the live payment harness so green means what it claims

_Origin: Codex review on PR #2065 (a dozen threads on `e2e-payments/`), all
verified against the code and none blocking the merge — the nightly run passes,
but each item is a way it could pass while proving less than its steps claim._

- **Partial-startup leak** (`cucumber/support/hooks.ts`): the `Before` hook
acquires server → tunnel → browser → sessions with no unwind; a rejection
after the first acquire leaks the app-server child into later scenarios.
Attach each resource as it is acquired or unwind in `finally`.
- **No failure notification before the summary** (`main.ts`): the terminal
`run().catch` reports but never calls `notifyFailure`, so missing credentials
and other pre-summary failures ping nothing.
- **Ambiguous click replay** (`browser.ts` `actOnControl`): when `ordinary()`
dispatched a submission but its navigation wait failed, the still-interactable
control is submitted again through the DOM fallback — a second POST on live
refund forms. Only fall back for failures proven to predate dispatch.
- **Chromium surviving teardown** (`browser.ts` `stop`): when both close paths
fail, the hook logs and resolves; the leaked browser keeps consuming runner
resources. Reject or kill the process.
- **Provider fetches without a bound** (`providers/shared.ts`): the harness's
own fetches carry no abort signal (the production transports now share
`PROVIDER_TIMEOUT_MS`); a hung sandbox read outlives its hook.
- **Malformed Stripe list answers** (`providers/stripe.ts`): both the
endpoint-list and refund-list reads default a missing `data` field to `[]`, so
a malformed 2xx can silently pass as "nothing there". Validate the documented
fields at the boundary.
- **Partial final refund passes** (`cucumber/steps/refund.ts`): the final
non-growth check only rejects amounts GREATER than the capture; a completed
400-of-2500 observation after the first check passes. Require exactness for
every completed final observation.
- **Broad refresh assertion** (`cucumber/steps/refund.ts`): the second,
observation-only refresh matches `/payment status/i`, which the rendered
button satisfies — an erroring refresh still passes. Assert the specific
outcome (the exact-amount first refresh is the model).
- **Protection not rechecked after refresh** (`cucumber/steps/refund.ts`): the
Refund/Delete-unavailable assertions run before the final refresh; a refresh
that re-enabled them would pass. Re-read the actions after it.
- **Vacuous webhook coverage** (`cucumber/steps/booking.ts`):
`holdFirstAppReturn` captures the return URL but holds nothing (interception
proved unreliable), so the browser return can book before "Stripe's signed
webhook confirms the payment" — the step then only polls the roster and passes
with the webhook broken. Assert independent webhook evidence (or rename the
claim).
- **memberB never verified** (`order-flow.ts`): `verifyComplexOrder` asserts
member A's two paths and the plain listing; member B's booking line and its £6
kit income are never checked in either the free or paid scenario.
- **Configured artifact directory ignored** (`main.ts`): cleanup, reports, and
the step summary use the hard-coded `e2e-payments/artifacts` while scenarios
write under `E2E_ARTIFACTS_DIR`; derive the root from config.

The coverage-exclusion thread from the same round (`scripts/run-tests.ts`) is
the same theme: the harness modules are excluded wholesale with the reasoning
recorded beside the list, and the pure helpers stay covered. The durable fix is
the one Codex names — push the env/config parsing behind injectable seams so
those branches get direct in-process tests — which is worth doing when the
harness is next open.
## Give the live payment harness direct tests through injectable seams

_Origin: the coverage-exclusion thread of the Codex review on PR #2065._

The harness modules under `e2e-payments/src/` are excluded from coverage
wholesale in `scripts/run-tests.ts`, with the reasoning recorded beside the
list; only the pure helpers stay covered. The durable fix is the one Codex names
— push the env/config parsing behind injectable seams so those branches get
direct in-process tests. The twelve behaviour findings from the same review
round are fixed; this remaining item is the test-architecture half.

## Deleting your own contact record also deletes your promotions opt-out

Expand Down
5 changes: 4 additions & 1 deletion e2e-payments/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,10 @@ visible submit control (`requestSubmit`) — never force, never `form.submit()`.
Some headless Chromium builds stop scheduling compositor frames after a form
POST, which stalls Playwright's stability wait; the harness detects that and
falls back to `requestSubmit` / a scripted click, which still runs browser
validation and the app's own event handlers.
validation and the app's own event handlers. A failed ordinary action is only
replayed through that fallback when a page-side witness proves the click never
dispatched — a post-dispatch failure rethrows, so a live refund form can never
be submitted twice by the fallback.

### Cleanup

Expand Down
2 changes: 2 additions & 0 deletions e2e-payments/specs/live-payment-providers.feature
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ Feature: Real sandbox payments finish safely
And Refresh is reachable while observation is unfinished
When the owner refreshes the payment without submitting Refund again
Then the provider's returned amount and Money refund count do not grow
And no second Refund action is available

@rule:payments.live-sumup-refund-is-safe @surface:admin @surface:public @surface:return @surface:webhook
Rule: A SumUp callback and refund reach a safe durable result
Expand All @@ -102,6 +103,7 @@ Feature: Real sandbox payments finish safely
And Refresh is reachable while observation is unfinished
When the owner refreshes the payment without submitting Refund again
Then the provider's returned amount and Money refund count do not grow
And no second Refund action is available

@rule:payments.live-invalidated-checkout-is-refunded @surface:admin @surface:public @surface:return @surface:webhook
Rule: A checkout invalidated while the visitor pays is retained and refunded
Expand Down
100 changes: 83 additions & 17 deletions e2e-payments/src/browser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { type Browser, chromium, type Locator, type Page } from "playwright";
import { browserLaunchOptions } from "#scripts/browser-options.ts";
import { config } from "./config.ts";
import { log } from "./log.ts";
import { repoRoot } from "./server.ts";
import { artifactsRoot } from "./server.ts";
import { pollUntil } from "./util.ts";

export interface BrowserSession {
Expand Down Expand Up @@ -80,14 +80,81 @@ export const hrefOf = async (
return href;
};

/** Whether the control can honestly be acted on at all. */
const interactable = async (locator: Locator): Promise<boolean> =>
(await locator.isVisible()) && (await locator.isEnabled());

/** The page-side shape the click witness stamps onto a control. */
type Witnessed = {
__e2eClickSeen?: boolean;
addEventListener: (
type: string,
listener: () => void,
options: { capture: boolean; once: boolean },
) => void;
};

/** Arm a page-side click witness on the control. The returned question is
* whether the click may have dispatched — and a witness that cannot answer
* (the element or its document is already gone, which navigation after a
* dispatched submission causes) says yes, never "safe to replay". */
const armClickWitness = async (
control: Locator,
): Promise<() => Promise<boolean>> => {
const armed = await control
.evaluate((element) => {
const witnessed = element as unknown as Witnessed;
witnessed.__e2eClickSeen = false;
witnessed.addEventListener(
"click",
() => {
witnessed.__e2eClickSeen = true;
},
{ capture: true, once: true },
);
})
.then(() => true)
.catch(() => false);
return () =>
armed
? control
.evaluate(
(element) =>
(element as unknown as Witnessed).__e2eClickSeen !== false,
)
.catch(() => true)
: Promise.resolve(true);
};

/** Arm the click witness and return the attempt that runs an ordinary action
* under it: true when the action finished, false only for the one
* replay-safe failure — the witness proves the click never dispatched and
* the control is still interactable. Every other failure rethrows: replaying
* a dispatched submission would act twice, and a second POST on a live
* refund form moves real money. */
const armWitnessedAttempt = async (
control: Locator,
): Promise<(ordinary: () => Promise<void>) => Promise<boolean>> => {
const mayHaveDispatched = await armClickWitness(control);
return async (ordinary) => {
try {
await ordinary();
return true;
} catch (error) {
if (await mayHaveDispatched()) throw error;
if (!(await interactable(control))) throw error;
return false;
}
};
};

export const launchAppBrowser = async (
baseUrl: string,
): Promise<AppBrowser> => {
log(`Launching Chromium (headless=${config.headless})…`);
const browser = await chromium.launch(
browserLaunchOptions(config.headless, config.chromiumExecutable),
);
const artifactsDir = join(repoRoot, "e2e-payments", config.artifactsDir);

const session = async (artifactPrefix: string): Promise<BrowserSession> => {
const context = await browser.newContext({ baseURL: baseUrl });
Expand Down Expand Up @@ -117,8 +184,8 @@ export const launchAppBrowser = async (
/** Save a screenshot AND the page HTML for debugging; the artifact prefix
* (the case id) keeps scenarios from overwriting each other's evidence. */
const dumpPage = async (label: string): Promise<void> => {
const png = join(artifactsDir, `${artifactPrefix}-${label}.png`);
const html = join(artifactsDir, `${artifactPrefix}-${label}.html`);
const png = join(artifactsRoot, `${artifactPrefix}-${label}.png`);
const html = join(artifactsRoot, `${artifactPrefix}-${label}.html`);
await page.screenshot({ fullPage: true, path: png }).catch(() => {});
await page
.content()
Expand Down Expand Up @@ -153,10 +220,6 @@ export const launchAppBrowser = async (
return framesLive;
};

/** Whether the control can honestly be acted on at all. */
const interactable = async (locator: Locator): Promise<boolean> =>
(await locator.isVisible()) && (await locator.isEnabled());

/** Act on a control through the page's own DOM APIs: `requestSubmit` for
* a form's submit control (which runs browser validation and the app's
* submit handlers), a scripted click for anything else. Typed without the
Expand Down Expand Up @@ -188,20 +251,17 @@ export const launchAppBrowser = async (
};

/** Act on a control like a person when the renderer is healthy, and
* through the page's own DOM APIs when it is frame-idle or the ordinary
* action times out on a still-interactable control. */
* through the page's own DOM APIs when it is frame-idle — or when the
* ordinary action provably failed before its click dispatched (see
* armWitnessedAttempt). */
const actOnControl = async (
control: Locator,
ordinary: () => Promise<void>,
): Promise<void> => {
const healthyRenderer = await framesAreLive();
if (healthyRenderer) {
try {
await ordinary();
return;
} catch (error) {
if (!(await interactable(control))) throw error;
}
const attempt = await armWitnessedAttempt(control);
if (await attempt(ordinary)) return;
}
await throughDomIfInteractable(control);
};
Expand Down Expand Up @@ -313,7 +373,13 @@ export const launchAppBrowser = async (
new Promise((resolve) => setTimeout(resolve, 5_000)),
]);
if (browser.isConnected()) {
log(" browser would not close; the runner will outlive it");
// Playwright exposes no process handle to kill, so the leak is
// raised instead of logged: the cleanup sweep reports it and fails
// an otherwise-green scenario rather than letting the zombie
// Chromium quietly consume the runner.
throw new Error(
"Chromium did not close after the bounded graceful close and CDP Browser.close",
);
}
}
},
Expand Down
Loading