diff --git a/.github/workflows/qa-smoke-preview-deploy.yml b/.github/workflows/qa-smoke-preview-deploy.yml index 2fdb50e92..a2ed8ab59 100644 --- a/.github/workflows/qa-smoke-preview-deploy.yml +++ b/.github/workflows/qa-smoke-preview-deploy.yml @@ -517,7 +517,7 @@ jobs: run_project missionary_status "missionary" "development-missionary" "${MISSIONARY_URL:-}" if [[ "${result}" == "FAIL" ]]; then - evidence="GitHub Actions artifact: pr-preview-smoke-playwright-artifacts; report paths: playwright-report/pr-preview-smoke-*; test result paths: test-results/pr-preview-smoke-*" + evidence="Sanitized diagnostics: playwright-report/pr-preview-smoke-*/sanitized/{index.html,results.json}. See the artifact-upload step for availability; raw test outputs are excluded." fi { @@ -534,13 +534,13 @@ jobs: with: name: pr-preview-smoke-playwright-artifacts path: | - playwright-report/pr-preview-smoke-* - test-results/pr-preview-smoke-* - if-no-files-found: ignore + playwright-report/pr-preview-smoke-*/sanitized/index.html + playwright-report/pr-preview-smoke-*/sanitized/results.json + if-no-files-found: error retention-days: 7 - name: Comment headless smoke QA result - if: steps.gate.outputs.should_run == 'true' + if: always() && steps.gate.outputs.should_run == 'true' env: ADMIN_STATUS: ${{ steps.smoke.outputs.admin }} ADMIN_URL: ${{ steps.deploy_admin.outputs.url }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0bbfbafc2..86cbdb9b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,12 @@ commit metadata does not. CODEOWNERS routes reviews but does not grant access. see `docs/guides/development/contributing.md`. - **Required local PR/push-readiness gate:** `bun run ci:preflight` (exact stages and focused debugging commands are documented in `docs/ci.md`). +- **Eve build boundary:** web builds and hosted admin previews emit unqualified + Eve artifacts without sandbox provisioning. Production services retain full + prewarming; use `bun run --cwd packages/eve-runtime build:full` in the approved + target environment for full qualification. Follow the + [build runbook](docs/guides/development/build-runbook.md#eve-artifacts-and-qualification) + and keep Release Off until the separate launch requirements are met. - **Production E2E:** `bun run test:e2e:production-gate` is the bounded release gate required for `production`; broader `bun run test:e2e` remains useful for local feature validation. diff --git a/README.md b/README.md index 7486deb42..6197aa458 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,13 @@ behavior; its source-owner amendments remain proposed. Use the per-app `dev:*` scripts when you only need one surface, or `bun run dev` / `bun run dev:all` when you need several (see root `package.json`). +`bun run build` and the `build:` commands compile Eve dependency artifacts +without provisioning sandboxes. Hosted admin previews use the same unqualified +artifact mode and leave Eve Release Off. Standalone Eve and production service +builds retain full qualification; see the +[build runbook](docs/guides/development/build-runbook.md#eve-artifacts-and-qualification) +before treating any preview as launch evidence. + ### AI Agent Guidance System Agent-oriented docs live under `docs/ai/`: diff --git a/apps/admin/eslint.config.mjs b/apps/admin/eslint.config.mjs index 95ca248ef..e80ec6797 100644 --- a/apps/admin/eslint.config.mjs +++ b/apps/admin/eslint.config.mjs @@ -5,4 +5,8 @@ import { designSystemConfig } from "@asym/eslint-config/design-system.mjs"; export default [ ...scopeWorkspaceConfig(nextjsConfig, import.meta.url), ...designSystemConfig({ workspace: "apps/admin" }), + { + // withEve emits deployment bundles here, not authored app/runtime source. + ignores: [".eve/vercel-services/**", ".vercel/output/**"], + }, ]; diff --git a/apps/admin/next.config.ts b/apps/admin/next.config.ts index b66473cef..cfa479208 100644 --- a/apps/admin/next.config.ts +++ b/apps/admin/next.config.ts @@ -101,4 +101,5 @@ const sentryConfig = withSentryConfig( export default withEve(sentryConfig, { eveRoot: `${WORKSPACE_ROOT}/packages/eve-runtime`, + eveBuildCommand: "bun run build:service", }); diff --git a/bun.lock b/bun.lock index 30a9e1363..fe2b5a53b 100644 --- a/bun.lock +++ b/bun.lock @@ -340,6 +340,7 @@ "dependencies": { "@asym/api": "workspace:*", "@asym/database": "workspace:*", + "@asym/env": "workspace:*", "@vercel/connect": "0.4.0", "ai": "7.0.31", "eve": "0.25.1", diff --git a/docs/guides/development/build-runbook.md b/docs/guides/development/build-runbook.md index 97b2e304b..91dbe577e 100644 --- a/docs/guides/development/build-runbook.md +++ b/docs/guides/development/build-runbook.md @@ -23,12 +23,47 @@ This runbook defines the canonical build workflow for the Bun + Turborepo monore Notes: -- Internal packages define `build` scripts as `tsc --noEmit` (type-check only, no JavaScript emit). +- Most shared packages define `build` as `tsc --noEmit` (type-check only). Eve + emits runtime artifacts; its qualification boundary is described below. - Example: `bunx turbo run build --filter=@asym/ui` now executes the `@asym/ui` package build task. - Turbo `build` caching tracks app artifacts (`.next/**`) and package/typecheck artifacts (`dist/**`, `*.tsbuildinfo`). - Cache output globs in `turbo.json` must never be able to match a package's own `node_modules`. Use package-relative globs (`*.tsbuildinfo`, `dist/*.tsbuildinfo`), not recursive ones (`**/*.tsbuildinfo`), and keep the `"!node_modules/**"` guard in `build.outputs` / `typecheck.outputs`. See [Turbo cache restore replaces workspace symlinks](#turbo-cache-restore-replaces-workspace-symlinks). - For troubleshooting, use Turbo filters directly: `bunx turbo run build --filter=@asym/`. +## Eve artifacts and qualification + +The generic web build planner sets `CORE_EVE_BUILD_MODE=artifacts` for its +children. Eve 0.25.1 then compiles with `--skip-sandbox-prewarm`; this applies to +both dependency and app phases without changing Turbo's dependency order. +Admin uses `bun run build:service` through the supported `withEve` build-command +option. This stable command selects artifact mode only when it executes on a +hosted preview. Production services retain the full SDK build, even when web +dependencies were compiled in artifact mode. The decision happens at service +execution because the SDK preserves previously generated service commands when +the same checkout builds another target. + +These are **unqualified Release-Off artifacts**. They can expose the deployed +candidate for inspection, but do not prove a sandbox template exists or permit +autonomous effects. Auth, governance, policy, audit and release admission remain +unchanged. The bundled Vercel runtime refuses a missing sandbox template; preview +health or web smoke success is not sandbox qualification. + +- `bun run --cwd packages/eve-runtime build` defaults to the full SDK build when + `CORE_EVE_BUILD_MODE` is unset. Unknown modes fail before the SDK starts. +- `bun run --cwd packages/eve-runtime build:artifacts` explicitly compiles only. +- `bun run --cwd packages/eve-runtime build:full` always selects the full SDK + build, including Vercel sandbox prewarming in a Vercel-targeted environment. + +Turbo hashes the mode and does not cache the Eve `build` task. An artifact build +or an earlier prewarm cannot be replayed as new provider qualification. A local +non-Vercel build remains local compilation under the SDK's own behavior. + +Before release, run full qualification in the correctly configured, authorized +target environment and collect the exact target-bound evidence required by the +[Eve launch runbook](../operations/eve-launch.md). Missing credentials, unavailable +governance, Release Off, or a denied prewarm remain qualification blockers. Do +not enable release to make CI pass or count artifact compilation as that proof. + ## Environment profiles ## 1) Default local profile (CI-equivalent) diff --git a/docs/guides/operations/eve-launch.md b/docs/guides/operations/eve-launch.md index 87e2100c6..61a8d4fe0 100644 --- a/docs/guides/operations/eve-launch.md +++ b/docs/guides/operations/eve-launch.md @@ -36,6 +36,15 @@ Do not copy credential values into the manifest. The panel must show **Runtime target: Configured** and **Release: Off**. If the governance state changes, produce a new manifest against the new state version. +Hosted web previews compile Eve artifacts without sandbox prewarming so this +Release-Off inspection can occur. These outputs are unqualified: a healthy web +preview or Eve health endpoint is not proof of sandbox availability or release +readiness. Before activation, full runtime/sandbox qualification and all evidence +below are still required for the exact target. Use the full build described in +the [build runbook](../development/build-runbook.md#eve-artifacts-and-qualification); +if credentials or governance deny prewarming, report that qualification blocker. +Never activate Eve or relax its policy merely to make a build pass. + ## 2. Collect launch evidence while release is off Build one `eve-launch-manifest-v1` JSON document using the schema exported by diff --git a/docs/qa/development-headless-smoke.md b/docs/qa/development-headless-smoke.md index 606709f9b..a2846b97a 100644 --- a/docs/qa/development-headless-smoke.md +++ b/docs/qa/development-headless-smoke.md @@ -86,7 +86,15 @@ The Playwright config is `playwright.development-smoke.config.ts`. It: (`QA__BASE_URL`, `VERCEL__AUTOMATION_BYPASS_SECRET`) - sends bypass via headers, not query params - runs headless Chromium, one worker -- writes report artifacts under `playwright-report/development-smoke/` +- writes HTML and JSON reports under `PLAYWRIGHT_REPORT_DIR`, defaulting to + `playwright-report/development-smoke/` +- writes bounded test evidence under + `PLAYWRIGHT_OUTPUT_DIR`, defaulting to `test-results/` + +The preview workflow sets both directories per surface so a later Playwright +invocation does not overwrite an earlier surface's failure evidence. Blank +overrides use the local defaults above. The suite-specific reporter preserves +the original failed exit and test status; it does not relax any assertion. The helpers in `tests/e2e/development-smoke/helpers.ts` cover: @@ -98,19 +106,47 @@ The helpers in `tests/e2e/development-smoke/helpers.ts` cover: ## How to view the report -```bash -bunx playwright show-report playwright-report/development-smoke -``` +Open `/sanitized/index.html`. This is a bounded diagnostic summary, +not Playwright's interactive trace/report viewer. ## Evidence captured on failure -For any failed test, Playwright keeps under -`playwright-report/development-smoke/`: - -- HTML report -- JSON report at `results.json` -- screenshot, trace, and video when retained by Playwright -- non-secret `evidence.json` attachments +For any failed test, Playwright keeps in the configured report and test-output +directories: + +- HTML summary +- JSON report at `/sanitized/results.json` +- test title, project, status and duration +- `evidence.json` with URL origin/path, page title/heading, and visible password + input count; known QA and bypass values are redacted +- the last 50 document/fetch/XHR response method/status/origin/path entries + observed during authentication, without request headers, cookies or bodies + +Known QA and bypass values are redacted before truncation in their raw/trimmed, +URI, URI-component, form-URL-encoded and UTF-8 base64/base64url forms (with or +without padding). Percent escapes may use either hex case. This is a bounded +set of representations, not detection of arbitrary transformations. Response +metadata describes only browser-visible requests; server-side profile reads +may be absent and an access-denied route alone does not establish its cause. + +URL queries and fragments are excluded. Raw trace, screenshot, video and the +automatic DOM error prompt are disabled for this credential-bearing suite: +traces retain bypass headers, authentication bodies and API arguments, and DOM +snapshots can retain password input values. The reporter replaces run-owned +test output (including generated error-context files) with the bounded +evidence above and never removes arbitrary attachment sources outside the +resolved run directories. It emits no assertion bodies or API-step text into +the HTML/JSON bundle. Reports and output directories must be separate and +must not be a workspace root or its ancestor. These checks resolve symlinks, +including the nearest existing parent of a new directory, before Playwright +startup. The reporter uses the checked canonical paths and checks them again +before cleanup. + +CI uploads only `sanitized/index.html` and `sanitized/results.json`; raw test +output is never part of the upload allowlist. If a reporter or worker fails +before producing a sanitized bundle, the missing-artifact step fails instead +of uploading leftovers. Successful cleanup is not a prerequisite for keeping +raw credentials out of uploaded artifacts. ## Safety Rules diff --git a/docs/qa/pr-preview-smoke.md b/docs/qa/pr-preview-smoke.md index b2c9b5115..d4b5c6567 100644 --- a/docs/qa/pr-preview-smoke.md +++ b/docs/qa/pr-preview-smoke.md @@ -115,6 +115,26 @@ Playwright. The preferred flow is that GitHub Actions deploys previews, runs Playwright, uploads sanitized failure artifacts, and comments the PASS/FAIL result for Claude to read. +Each surface invocation also sets `PLAYWRIGHT_REPORT_DIR` to +`playwright-report/pr-preview-smoke-` and `PLAYWRIGHT_OUTPUT_DIR` to +`test-results/pr-preview-smoke-`. The development smoke configuration +uses these paths for its bounded HTML/JSON summaries and redacted test evidence, +preserving earlier surfaces' evidence. The upload step allowlists only +`playwright-report/pr-preview-smoke-*/sanitized/index.html` and +`playwright-report/pr-preview-smoke-*/sanitized/results.json`; it never uploads +raw test output and fails if no sanitized file exists. Raw trace/media, DOM +snapshots and API-step/assertion bodies are excluded because they can retain +QA credentials and Vercel bypass values. URL +queries and fragments are removed; test status, title, duration, and safe +origin/path/title/heading remain available. Up to 50 authentication response +method/status/origin/path entries show browser-visible responses without +recording headers, cookies or bodies. Server-side profile reads may be absent; +these entries alone do not establish a profile, membership, tenant or role +failure. See the +[artifact boundary](./development-headless-smoke.md#evidence-captured-on-failure). +An artifact records the failed check; it does not turn a failed smoke test into +a pass. + ## Rerun Method Use one of these: diff --git a/eslint.config.mjs b/eslint.config.mjs index 2786ecd3b..4f084a003 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -181,6 +181,9 @@ const eslintConfig = defineConfig([ ignores: [ ".next/**", "**/.next/**", + // SDK-generated admin service bundles; authored Eve/app code stays linted. + "apps/admin/.eve/vercel-services/**", + "apps/admin/.vercel/output/**", "out/**", "**/out/**", "build/**", diff --git a/openspec/changes/separate-eve-preview-artifacts-from-qualification/design.md b/openspec/changes/separate-eve-preview-artifacts-from-qualification/design.md new file mode 100644 index 000000000..b53305c0b --- /dev/null +++ b/openspec/changes/separate-eve-preview-artifacts-from-qualification/design.md @@ -0,0 +1,69 @@ +# Design + +The web build planner passes `CORE_EVE_BUILD_MODE=artifacts` to its child +commands while retaining Turbo's dependency graph. The Eve workspace build +dispatcher defaults to full compilation/prewarming and rejects unknown modes. +Explicit `build:artifacts` and `build:full` commands remain available. + +Admin's existing `withEve` integration uses one stable `build:service` command. +That dispatcher selects artifact mode only when it executes on a hosted preview. +The SDK preserves existing generated service commands, so choosing the mode only +when generating config could carry a preview skip into a later production build. +Service execution ignores the generic web artifact variable; production and +ordinary standalone commands retain the SDK's full build. A production target +overrides a conflicting preview environment marker. The dispatcher uses the +canonical deployment-environment helpers so case and whitespace cannot hide a +production target. Hosted core-development and legacy staging retain artifact +mode; an explicit local development target retains full mode. + +Both named build commands select their explicit mode through the same dispatcher, +overriding an inherited generic build mode. Full mode rejects the SDK's +skip-prewarm flag before starting Eve, while ordinary SDK arguments still pass +through. Explicit selectors cannot be combined with a forwarded service or +another mode selector. The Node 24 dispatcher imports the declared environment workspace +directly; it does not introduce another environment classifier. + +Turbo hashes the mode. The Eve build task does not cache provider qualification: +an artifact build or a prior successful prewarm cannot stand in for a fresh +required full build. All auth, governance, sandbox and release code is unchanged. + +Admin's generated `.eve/vercel-services` and `.vercel/output` bundles are excluded +from authored-source linting and data-boundary scans. Regression coverage checks +the exact output paths and confirms admin app/config and Eve agent/runtime source +remain checked. Actual scanner CLI fixtures reject raw database imports in +authored admin files, neighboring `.eve`/`.vercel` files, and another app's +same-named directory; retired CRM markers in authored Eve runtime still fail. + +Installed Eve 0.25.1 source shows that skipping prewarm still emits app and +workflow functions. Its bundled Vercel runtime refuses a missing template rather +than prewarming it on demand. An isolated SDK fixture verified generated service +output and health without credentials or network access. Actual Core validation +must additionally exercise the package dispatcher and generated service. + +Release-Off previews are unqualified artifacts. Later full sandbox/runtime +qualification and target-bound launch evidence remain required. If governance +denies that qualification, the denial remains a blocker; this change supplies no +alternate authorization path. No API, database, migration or runtime policy +change is part of this repair. + +## Shared-context validation follow-up + +The canonical full gate exposed a pre-existing relation-ID false positive: +the valid UUID `01234567-8910-4111-8123-456789012345` contains a substring matching +the payment-number detector. A generated related claim ID could therefore +reject an otherwise valid disagreement. This is the narrow repair already +preserved from #1862 in #1905's reviewed integration candidate. + +After the existing UUID schema validation, sensitive-content scanning excludes +only the top-level `relatedClaimIds` metadata. It still scans claim values, +provenance, evidence and other content, including UUID-shaped payment text. +The original IDs are retained in the claim and still require visible existing +claims for the same field, tenant and root run. Invalid IDs and relationships +remain rejected. This restores the accepted structured context and disagreement +preservation requirements in `eve-subagent-catalog-shared-run-context`; it +does not grant new authority or change any release gate. + +The deterministic relation-ID regression and UUID-shaped-content rejection +control travel with this repair. #1862's design packs and the remaining #1905 +work are still pending separate integration; this small repair does not +establish either PR's full supersession. diff --git a/openspec/changes/separate-eve-preview-artifacts-from-qualification/preview-diagnostics.md b/openspec/changes/separate-eve-preview-artifacts-from-qualification/preview-diagnostics.md new file mode 100644 index 000000000..016374c3d --- /dev/null +++ b/openspec/changes/separate-eve-preview-artifacts-from-qualification/preview-diagnostics.md @@ -0,0 +1,89 @@ +# Preview smoke diagnostic follow-up + +The first hosted verification of PR #1915 deployed all three preview surfaces, +then failed each authenticated marker check. The workflow selected per-surface +report/output directories, while the Playwright configuration ignored those +variables. No matching artifacts were uploaded. This follow-up restores that +existing handoff contract; it does not change app authentication, authorization, +Eve admission, smoke assertions, or the failed test outcome. + +A local synthetic control also confirmed that ordinary Playwright traces, +HTML report payloads and error-context files retain QA/bypass values. The +credential-bearing smoke suite therefore publishes only bounded diagnostics: +test title, project, failed status, duration, sanitized URL origin/path, +redacted page title/heading, and visible password-input count. Automatic raw +media/DOM capture and API-step/assertion bodies are excluded. Redaction covers +known raw/trimmed, URI, URI-component, form-URL-encoded and UTF-8 base64/base64url +values, including optional padding and either percent-escape hex case. It does +not claim to detect arbitrary transformations. + +Cleanup is limited to canonical run report/output directories; external +attachment sources remain untouched. Validation resolves existing symlinks and +the nearest existing parent of new paths before checking workspace/root and +report/output overlap. Playwright and the reporter use these canonical paths; +the reporter revalidates them before cleanup. + +The helper also observes the last 50 document/fetch/XHR responses during the +authentication attempt. Only method, status, origin and path survive into the +bundle; query/fragment, headers, cookies and bodies are excluded, and known QA +identity values are redacted. This provides browser-visible response evidence +without a debug API or privileged query. Server-side profile reads may be +absent; these entries alone cannot establish why an access-denied page appears. + +Six browser-free unit checks invoke the actual pinned Playwright CLI. +Deliberately failed auth-shaped requests stay failed, separate surface bundles +survive consecutive runs, blank overrides preserve local defaults, raw copies +are removed, and an attachment source outside the output root survives. The +entire final HTML/JSON/output bundle is checked for synthetic credential values +and opaque/compressed artifacts. The encoded-canary case places URL/form/base64 +representations in retained URL paths, title, heading and network paths, then +verifies their redaction and preservation of failed status. + +The workflow uploads only the dedicated sanitized HTML/JSON paths, never raw +test output. A simulated reporter completion failure leaves a raw dummy secret +behind but produces no uploadable file. Two cases also reject report/output +workspace roots before Playwright startup can clear them. Eight additional +recording-filesystem checks cover symlink aliases to the workspace, its ancestor, +equal or nested new report/output paths, dangling links, paths changed before +cleanup, and valid separate paths. All six CLI +cases, eight path checks and eleven workflow contract tests pass; missing +sanitized artifacts fail the upload step rather than claiming an artifact exists. + +A separate actual Chromium comparison uses dummy credentials and a localhost +server. The old configuration retains all five canaries across 27 files, +including recursively decoded ZIP/base64 report content. +The candidate preserves the failed marker check and emits four readable files +without any canary. This is artifact-handling proof, not successful live login. + +Run `36479535225` on head `d76f2390` successfully deployed all three previews +and uploaded the six allowlisted reports. Each test observed a token endpoint +200 response followed by a protected-route redirect. Admin and missionary +finished at `/no-access` with the `No access` heading; donor finished at its +public `/`. These are per-test browser observations, unlike earlier aggregate +route counts. No browser profile/RPC response was captured; server-side +profile, membership, tenant or role cause remains unproven. + +The existing role resolver collapses profile errors, no visible profile, +membership RPC errors and exceptions into null. Preview-only server diagnostics +now distinguish those existing decisions and a resolved-but-disallowed role +without adding a request, endpoint or permission rule. Canonical +`@asym/env/target-env` helpers allow only preview classification and reject +protected targets, including contradictory production/preview signals. + +The emitted object has only literal `event`, `stage`, `outcome` and `code` +values. The event is `auth_access_diagnostic`; stages are `profile_read`, +`membership_read`, `resolver` and `role_gate`. Outcomes are `query_failed`, +`no_visible_profile`, `exception` and `role_denied`. Codes are limited to +`42501`, `42P01`, `42883`, `PGRST106`, `PGRST116`, `PGRST202`, `PGRST301`, +`other` and null. It never serializes identities, tenants, role values, raw +error messages/details/hints, headers, cookies or credentials. No-visible-profile +does not distinguish a missing row from an RLS-hidden row. Diagnostic property +or log-sink failures cannot change the original auth result. Successful +resolution and protected/non-preview environments remain silent. + +Focused tests retain the same null/role results and both middleware redirect +sites, including refreshed cookies, while discriminating failure stages and +challenging hostile error properties, encoded canaries and a throwing log sink. +Task 2.5 remains open: the reviewed candidate still needs normal integration +gates and a new authorized Actions run using its existing opaque QA secrets. +The actual smoke assertions must pass before claiming hosted QA completion. diff --git a/openspec/changes/separate-eve-preview-artifacts-from-qualification/proposal.md b/openspec/changes/separate-eve-preview-artifacts-from-qualification/proposal.md new file mode 100644 index 000000000..052903f40 --- /dev/null +++ b/openspec/changes/separate-eve-preview-artifacts-from-qualification/proposal.md @@ -0,0 +1,44 @@ +# Separate Eve preview artifacts from runtime qualification + +Tracking issue: [AL-1913](https://github.com/Asymmetric-al/core/issues/1913). + +## Why + +Admin preview builds traverse the Eve workspace dependency and then build the +generated `withEve` service. Both invoke sandbox prewarming, which correctly +refuses unavailable or Release-Off governance. This prevents the deployed, +Release-Off candidate required by the launch runbook from being inspected. + +## What changes + +- Give generic web builds an explicit Eve artifact mode and use Eve 0.25.1's + supported `--skip-sandbox-prewarm` option for hosted preview services. +- Preserve ordinary standalone and production service builds with prewarming. +- Keep artifact mode distinct in build caching and describe its unqualified + status in the build and launch runbooks. +- Restore per-surface preview diagnostics with bounded, redacted artifacts while + preserving every existing smoke assertion and failed result. +- Distinguish preview access-resolution failures with fixed server diagnostic + stages and allowlisted error codes, preserving every authorization decision + and keeping identities, roles, credentials and raw errors out of logs. +- Carry forward the reviewed shared-context relation-ID repair preserved from + #1862 in #1905, so valid UUIDs do not intermittently fail the full gate's + sensitive-content check. + +## Scope and boundaries + +This changes build orchestration, preview diagnostic handling, and a narrow +shared-context validation defect. Schema-validated relation IDs remain subject +to visible-claim, field, tenant and run checks; content remains subject to the +existing sensitive-data rules. It does not activate Eve, change governance or +effect admission, authorize provisioning, supply credentials, or qualify a +preview for release. A passing preview is not sandbox or launch proof. + +## Capability + +- `eve-runtime-foundation`: separate compilation from runtime qualification. + +## Rollback + +Revert this focused change to restore full prewarming on web builds. The +existing fail-closed preview build failure returns; no data migration is needed. diff --git a/openspec/changes/separate-eve-preview-artifacts-from-qualification/specs/eve-runtime-foundation/spec.md b/openspec/changes/separate-eve-preview-artifacts-from-qualification/specs/eve-runtime-foundation/spec.md new file mode 100644 index 000000000..4d55ba51a --- /dev/null +++ b/openspec/changes/separate-eve-preview-artifacts-from-qualification/specs/eve-runtime-foundation/spec.md @@ -0,0 +1,62 @@ +## ADDED Requirements + +### Requirement: Web Build Artifacts Are Separate From Eve Runtime Qualification + +Generic web builds MUST compile their Eve dependency without provisioning a +sandbox. Hosted preview Eve services MUST support the same explicit artifact +mode. These outputs MUST be treated as unqualified Release-Off previews and MUST +NOT activate Eve, change effect admission, or satisfy sandbox/launch evidence. +Build caches MUST distinguish artifact mode and MUST NOT reuse a cached result +as full sandbox qualification. Ordinary standalone and production service builds +MUST retain the full SDK prewarm path and propagate its failures. +Authored-source verification MUST exclude only the SDK's exact generated service +and deployment output directories, while retaining checks on authored code and +same-named directories outside those paths. + +#### Scenario: A Release-Off preview compiles without provisioning + +- **GIVEN** the web preview requires Eve service output but release is off +- **WHEN** the web dependency and generated preview service are built +- **THEN** complete app/workflow artifacts are emitted without sandbox prewarm +- **AND** existing runtime governance continues to deny unauthorized effects +- **AND** a successful preview is not represented as full Eve qualification + +#### Scenario: Full qualification remains mandatory + +- **GIVEN** an artifact build succeeded without a qualified sandbox template +- **WHEN** a standalone full build or production service build runs +- **THEN** the normal SDK prewarm path still runs and any denial fails the build +- **AND** artifact or prior cache success cannot substitute for that qualification +- **AND** full runtime verification and target-bound launch evidence remain + required before any authorized release activation + +#### Scenario: Generated service output is reused for another target + +- **GIVEN** a checkout generated an Eve service during a preview build +- **WHEN** that service command executes for a production target +- **THEN** it selects full SDK qualification using the current target +- **AND** neither stale preview configuration nor an inherited web artifact + variable can skip prewarming + +#### Scenario: Generated server bundles coexist with authored app source + +- **GIVEN** the SDK emitted admin `.eve/vercel-services` or `.vercel/output` bundles +- **WHEN** lint and data-boundary verification run after the build +- **THEN** they exclude those generated bundles without classifying server dependencies as authored browser imports +- **AND** forbidden database imports or retired CRM references in authored source still fail verification + +#### Scenario: Production signals require canonical normalization + +- **GIVEN** the current target has production casing or whitespace accepted by the canonical environment model +- **WHEN** the generated service dispatcher selects its build mode +- **THEN** production still requires full prewarming even with a conflicting preview marker +- **AND** local development stays full while hosted core-development and legacy staging use artifacts + +#### Scenario: Explicit full commands receive a conflicting skip flag + +- **GIVEN** an ordinary full build, production service or named full command +- **WHEN** the caller supplies the SDK skip-prewarm flag +- **THEN** the dispatcher fails before starting Eve +- **AND** an inherited artifact-mode variable cannot change the named full command +- **AND** a forwarded service or artifact selector cannot override that explicit full mode +- **AND** ordinary SDK arguments remain supported diff --git a/openspec/changes/separate-eve-preview-artifacts-from-qualification/tasks.md b/openspec/changes/separate-eve-preview-artifacts-from-qualification/tasks.md new file mode 100644 index 000000000..fe0488721 --- /dev/null +++ b/openspec/changes/separate-eve-preview-artifacts-from-qualification/tasks.md @@ -0,0 +1,16 @@ +## 1. Reproduce and specify + +- [x] 1.1 Trace the preview failure and installed SDK build/service paths. +- [x] 1.2 Verify isolated SDK artifacts without credentials or network access. +- [x] 1.3 Add failing web-planner and preview-service regression tests. + +## 2. Implement and validate + +- [x] 2.1 Add explicit artifact orchestration while retaining full qualification. +- [x] 2.2 Bind cache behavior to mode and document the qualification boundary. +- [x] 2.3 Pass focused tests, formatting, lint/typecheck and strict OpenSpec checks. +- [x] 2.4 Verify actual Core artifacts and full-build denial in isolation. +- [ ] 2.5 Complete independent review, canonical preflight and required preview CI. +- [x] 2.6 Verify review findings adversarially and cover normalized targets and full-mode argument rejection through dispatcher and real CLI regression tests. +- [x] 2.7 Preserve schema-validated relation UUIDs while retaining sensitive-content and tenant/run/field rejection, with deterministic regression controls. +- [x] 2.8 Distinguish preview profile/RPC failures from no-visible-profile and role denial with literal-only diagnostics, proving protected-target silence and unchanged failure/redirect/cookie behavior. diff --git a/packages/auth/access-diagnostics.ts b/packages/auth/access-diagnostics.ts new file mode 100644 index 000000000..e247c3af9 --- /dev/null +++ b/packages/auth/access-diagnostics.ts @@ -0,0 +1,72 @@ +import { + isProtectedDeployment, + resolveDeploymentEnvironment, +} from "@asym/env/target-env"; + +const DIAGNOSTICS = { + profile_query_failed: { stage: "profile_read", outcome: "query_failed" }, + no_visible_profile: { stage: "profile_read", outcome: "no_visible_profile" }, + membership_query_failed: { + stage: "membership_read", + outcome: "query_failed", + }, + resolver_exception: { stage: "resolver", outcome: "exception" }, + role_denied: { stage: "role_gate", outcome: "role_denied" }, +} as const; + +const SAFE_ERROR_CODES = [ + "42501", + "42P01", + "42883", + "PGRST106", + "PGRST116", + "PGRST202", + "PGRST301", +] as const; + +function safeErrorCode(error: unknown) { + try { + const code = + error && typeof error === "object" && "code" in error + ? error.code + : undefined; + return SAFE_ERROR_CODES.find((allowed) => allowed === code) ?? "other"; + } catch { + // Even an unreadable error property must not affect access resolution. + return "other"; + } +} + +/** Fixed preview metadata only; diagnostics never participate in authorization. */ +export function reportAccessDiagnostic( + kind: keyof typeof DIAGNOSTICS, + error?: unknown, +): void { + try { + const environment = { + VERCEL_ENV: process.env.VERCEL_ENV, + VERCEL_TARGET_ENV: process.env.VERCEL_TARGET_ENV, + }; + if ( + resolveDeploymentEnvironment(environment) !== "preview" || + isProtectedDeployment(environment) || + !Object.hasOwn(DIAGNOSTICS, kind) + ) { + return; + } + + const diagnostic = DIAGNOSTICS[kind]; + console.warn({ + event: "auth_access_diagnostic", + stage: diagnostic.stage, + outcome: diagnostic.outcome, + code: + diagnostic.outcome === "query_failed" || + diagnostic.outcome === "exception" + ? safeErrorCode(error) + : null, + }); + } catch { + // A failing log sink must preserve the caller's original auth result. + } +} diff --git a/packages/auth/middleware.ts b/packages/auth/middleware.ts index 4c9446f02..e8b6e4931 100644 --- a/packages/auth/middleware.ts +++ b/packages/auth/middleware.ts @@ -5,6 +5,7 @@ import { import { createServerClient } from "@supabase/ssr"; import { NextResponse, type NextRequest } from "next/server"; +import { reportAccessDiagnostic } from "./access-diagnostics"; import { safeNextParam } from "./demo-login"; import { assertSupabaseDatasourceAllowedForE2EBypass, @@ -311,6 +312,9 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions = {}) { : null; if (!roleSnapshot || !hasAnyRole(roleSnapshot, allowedRoles)) { + if (roleSnapshot) { + reportAccessDiagnostic("role_denied"); + } return redirectWithCookies( buildRedirectUrl(request, unauthorizedRedirectTo), supabaseResponse, @@ -356,6 +360,9 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions = {}) { : null; if (!roleSnapshot || !hasAnyRole(roleSnapshot, allowedRoles)) { + if (roleSnapshot) { + reportAccessDiagnostic("role_denied"); + } return redirectWithCookies( buildRedirectUrl(request, unauthorizedRedirectTo), supabaseResponse, diff --git a/packages/auth/resolve-user-role.ts b/packages/auth/resolve-user-role.ts index 278ae615a..0f0755894 100644 --- a/packages/auth/resolve-user-role.ts +++ b/packages/auth/resolve-user-role.ts @@ -1,3 +1,4 @@ +import { reportAccessDiagnostic } from "./access-diagnostics"; import { DEMO_TENANT_ID } from "./constants"; import type { SupabaseUserRoleReader } from "./middleware"; @@ -49,6 +50,10 @@ export async function resolveUserRoleFromDatabase({ } | null; if (profileError || !profile) { + reportAccessDiagnostic( + profileError ? "profile_query_failed" : "no_visible_profile", + profileError, + ); return null; } @@ -73,7 +78,8 @@ export async function resolveUserRoleFromDatabase({ } return { profileRole, memberships }; - } catch { + } catch (error) { + reportAccessDiagnostic("resolver_exception", error); return null; } } @@ -100,6 +106,7 @@ async function loadActiveMemberships( }); if (error) { + reportAccessDiagnostic("membership_query_failed", error); return null; } diff --git a/packages/eve-runtime/package.json b/packages/eve-runtime/package.json index f39a4be5a..ffdc71672 100644 --- a/packages/eve-runtime/package.json +++ b/packages/eve-runtime/package.json @@ -10,7 +10,10 @@ "./launch-watchdog": "./src/launch/watchdog.ts" }, "scripts": { - "build": "eve build", + "build": "node scripts/build.mjs", + "build:artifacts": "node scripts/build.mjs --artifacts", + "build:full": "node scripts/build.mjs --full", + "build:service": "node scripts/build.mjs --service", "eval": "eve eval smoke specialists --strict --max-concurrency 1", "info": "eve info", "lint": "eslint agent evals src", @@ -19,6 +22,7 @@ "dependencies": { "@asym/api": "workspace:*", "@asym/database": "workspace:*", + "@asym/env": "workspace:*", "@vercel/connect": "0.4.0", "ai": "7.0.31", "eve": "0.25.1", diff --git a/packages/eve-runtime/scripts/build.mjs b/packages/eve-runtime/scripts/build.mjs new file mode 100644 index 000000000..87fd50aa2 --- /dev/null +++ b/packages/eve-runtime/scripts/build.mjs @@ -0,0 +1,95 @@ +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + isProductionDeployment, + normalizeDeploymentEnvironmentName, + resolveDeploymentEnvironment, +} from "@asym/env/target-env"; + +const require = createRequire(import.meta.url); +const EVE_BINARY = path.join( + path.dirname(require.resolve("eve/package.json")), + "bin/eve.js", +); +const PACKAGE_ROOT = fileURLToPath(new URL("..", import.meta.url)); + +function isHostedArtifactServiceBuild(environment) { + return ( + environment.VERCEL === "1" && + normalizeDeploymentEnvironmentName(environment.VERCEL_ENV) === "preview" && + !isProductionDeployment(environment) && + resolveDeploymentEnvironment(environment) !== "development" + ); +} + +export function runEveBuild({ + environment = process.env, + args = [], + service = false, + mode: explicitMode, + spawn = spawnSync, +} = {}) { + // Generated Vercel service config can survive a later build in this checkout. + // Decide from the service's current target, never from a saved preview command + // or the artifact mode inherited from the generic web dependency build. + const mode = + explicitMode ?? + (service + ? isHostedArtifactServiceBuild(environment) + ? "artifacts" + : "full" + : (environment.CORE_EVE_BUILD_MODE ?? "full")); + if (mode !== "full" && mode !== "artifacts") { + throw new Error("CORE_EVE_BUILD_MODE must be full or artifacts."); + } + if ( + args.some((arg) => ["--full", "--artifacts", "--service"].includes(arg)) + ) { + throw new Error("Eve build mode selectors cannot be combined."); + } + if (mode === "full" && args.includes("--skip-sandbox-prewarm")) { + throw new Error( + "--skip-sandbox-prewarm is only supported in artifacts mode.", + ); + } + + const buildArgs = [EVE_BINARY, "build"]; + if (mode === "artifacts") { + console.log("Eve artifact build: sandbox qualification is still required."); + buildArgs.push("--skip-sandbox-prewarm"); + } + buildArgs.push(...args); + + const result = spawn(process.execPath, buildArgs, { + cwd: PACKAGE_ROOT, + env: environment, + shell: false, + stdio: "inherit", + }); + if (result.error) throw result.error; + return result.status ?? 1; +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const args = process.argv.slice(2); + const service = args[0] === "--service"; + const mode = + args[0] === "--full" + ? "full" + : args[0] === "--artifacts" + ? "artifacts" + : undefined; + if (service || mode) args.shift(); + process.exitCode = runEveBuild({ args, service, mode }); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + } +} diff --git a/packages/eve-runtime/turbo.json b/packages/eve-runtime/turbo.json index d1f607cc1..eb4fc7a82 100644 --- a/packages/eve-runtime/turbo.json +++ b/packages/eve-runtime/turbo.json @@ -3,6 +3,7 @@ "extends": ["//"], "tasks": { "build": { + "cache": false, "outputs": [] }, "typecheck": { diff --git a/playwright.development-smoke.config.ts b/playwright.development-smoke.config.ts index b6f42f97c..6578ed992 100644 --- a/playwright.development-smoke.config.ts +++ b/playwright.development-smoke.config.ts @@ -1,5 +1,9 @@ +import { resolve } from "node:path"; + import { defineConfig, devices } from "@playwright/test"; +import { assertSmokeArtifactDirectories } from "./tests/e2e/development-smoke/safe-reporter"; + /** * Headless development/PR-preview smoke tests. * @@ -62,8 +66,25 @@ function surfaceProject(name: `development-${SurfaceKey}`, key: SurfaceKey) { }; } +const requestedReportDirectory = resolve( + readEnv("PLAYWRIGHT_REPORT_DIR") ?? "playwright-report/development-smoke", +); +const requestedOutputDirectory = resolve( + readEnv("PLAYWRIGHT_OUTPUT_DIR") ?? "test-results", +); +// Validate before Playwright can clear its output directory during startup. +const { reportDirectory, outputDirectories } = assertSmokeArtifactDirectories( + requestedReportDirectory, + [requestedOutputDirectory], +); + +// Pinned Playwright's automatic error prompt otherwise snapshots input values. +// The suite reporter retains bounded, redacted diagnostics instead. +process.env.PLAYWRIGHT_NO_COPY_PROMPT = "1"; + export default defineConfig({ testDir: "./tests/e2e/development-smoke", + outputDir: outputDirectories[0], fullyParallel: false, forbidOnly: !!process.env.CI, retries: 0, @@ -71,21 +92,16 @@ export default defineConfig({ timeout: 90_000, expect: { timeout: 15_000 }, reporter: [ - ["list"], - [ - "html", - { outputFolder: "playwright-report/development-smoke", open: "never" }, - ], [ - "json", - { outputFile: "playwright-report/development-smoke/results.json" }, + resolve(__dirname, "tests/e2e/development-smoke/safe-reporter.ts"), + { outputFolder: reportDirectory }, ], ], use: { headless: true, - trace: "retain-on-failure", - screenshot: "only-on-failure", - video: "retain-on-failure", + trace: "off", + screenshot: "off", + video: "off", navigationTimeout: 60_000, actionTimeout: 20_000, }, diff --git a/scripts/verify/ci-build.mjs b/scripts/verify/ci-build.mjs index ca4e06c64..704ab81a0 100644 --- a/scripts/verify/ci-build.mjs +++ b/scripts/verify/ci-build.mjs @@ -103,11 +103,13 @@ function createRunWithCiEnvStep(label, command, args) { } function createBuildStep(label, command, args, { strict = false } = {}) { + // Web dependency compilation is not authorization to provision Eve sandboxes. + const env = { CORE_EVE_BUILD_MODE: "artifacts" }; if (strict) { - return { label, command, args }; + return { label, command, args, env }; } - return createRunWithCiEnvStep(label, command, args); + return { ...createRunWithCiEnvStep(label, command, args), env }; } export function getSharedPackageBuildSteps({ @@ -185,12 +187,12 @@ export function getRequestedApps(args = [], apps = NEXT_APPS) { return [requestedApp]; } -function run(command, args, label) { +function run(command, args, label, environment) { console.log(`==> CI build: ${label}`); const result = spawnSync(command, args, { cwd: REPO_ROOT, stdio: "inherit", - env: process.env, + env: { ...process.env, ...environment }, }); if (result.error) { @@ -315,7 +317,7 @@ function main(args = process.argv.slice(2)) { apps: requestedApps, })) { clearStaleNextLocks(); - run(step.command, step.args, step.label); + run(step.command, step.args, step.label, step.env); clearStaleNextLocks(); } @@ -327,7 +329,7 @@ function main(args = process.argv.slice(2)) { // scripts/repair-workspace-links.mjs. repairAndLogWorkspaceLinks(); clearStaleNextLocks(); - run(step.command, step.args, step.label); + run(step.command, step.args, step.label, step.env); clearStaleNextLocks(); } diff --git a/scripts/verify/data-boundary-check.mjs b/scripts/verify/data-boundary-check.mjs index 182c1f43d..8c4a12877 100644 --- a/scripts/verify/data-boundary-check.mjs +++ b/scripts/verify/data-boundary-check.mjs @@ -52,6 +52,9 @@ const SKIP_REPO_RELATIVE_DIRECTORIES = new Set([ "packages/eve-runtime/.eve", "packages/eve-runtime/.nitro", "packages/eve-runtime/.output", + // withEve copies server/runtime dependencies into these deployment bundles. + "apps/admin/.eve/vercel-services", + "apps/admin/.vercel/output", ]); const RETIRED_TWENTY_RUNTIME_MARKERS = [ "TWENTY_API_URL", @@ -76,6 +79,8 @@ const IGNORED_GENERATED_RUNTIME_PREFIXES = [ "packages/eve-runtime/.eve/", "packages/eve-runtime/.nitro/", "packages/eve-runtime/.output/", + "apps/admin/.eve/vercel-services/", + "apps/admin/.vercel/output/", ]; function toRepoRelative(filePath) { diff --git a/tests/e2e/development-smoke/helpers.ts b/tests/e2e/development-smoke/helpers.ts index 7d3b85e42..618cd073b 100644 --- a/tests/e2e/development-smoke/helpers.ts +++ b/tests/e2e/development-smoke/helpers.ts @@ -14,6 +14,31 @@ import { */ export type AuthMarker = (page: Page) => Locator; +type AuthResponse = { method: string; status: number; url: string }; +const authResponses = new WeakMap(); + +/** Observe only bounded request metadata, never headers, cookies or bodies. */ +export function observeAuthenticationRequests(page: Page): void { + if (authResponses.has(page)) return; + const responses: AuthResponse[] = []; + authResponses.set(page, responses); + page.on("response", (response) => { + const request = response.request(); + if (!["document", "fetch", "xhr"].includes(request.resourceType())) return; + try { + const url = new URL(response.url()); + responses.push({ + method: request.method(), + status: response.status(), + url: `${url.origin}${url.pathname}`, + }); + if (responses.length > 50) responses.shift(); + } catch { + // Non-URL responses cannot contribute safe navigation evidence. + } + }); +} + /** * Shared helpers for headless development-deployment smoke tests. * @@ -274,6 +299,7 @@ export async function ensureAuthenticated( authenticatedMarker?: AuthMarker; }, ): Promise { + observeAuthenticationRequests(page); const options = typeof pathOrOptions === "string" ? { targetPath: pathOrOptions } @@ -351,11 +377,11 @@ export async function assertNoErrorBanner(page: Page): Promise { /** * Capture compact, non-secret evidence on failure. Attaches: - * - current URL + * - current URL origin/path (no query or fragment) * - page title * - main heading text (first h1/h2/h3) * - count of password inputs still in the DOM - * - screenshot (Playwright also retains one via `screenshot: only-on-failure`) + * - last 50 document/fetch/XHR method/status/origin/path observations * * Never reads or attaches input values. Call from a `test.afterEach` block * when `testInfo.status !== testInfo.expectedStatus`. @@ -368,7 +394,7 @@ export async function collectFailureEvidence( const evidence = await page .evaluate(() => ({ - url: location.href, + url: `${location.origin}${location.pathname}`, title: document.title, heading: document.querySelector("h1,h2,h3")?.textContent?.trim().slice(0, 120) ?? @@ -388,7 +414,11 @@ export async function collectFailureEvidence( })); await testInfo.attach("evidence.json", { - body: JSON.stringify(evidence, null, 2), + body: JSON.stringify( + { ...evidence, network: authResponses.get(page) ?? [] }, + null, + 2, + ), contentType: "application/json", }); } diff --git a/tests/e2e/development-smoke/safe-reporter.ts b/tests/e2e/development-smoke/safe-reporter.ts new file mode 100644 index 000000000..6c7089c64 --- /dev/null +++ b/tests/e2e/development-smoke/safe-reporter.ts @@ -0,0 +1,331 @@ +import { createHash } from "node:crypto"; +import { + lstatSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { + basename, + dirname, + isAbsolute, + join, + parse, + relative, + resolve, + sep, +} from "node:path"; + +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from "@playwright/test/reporter"; + +const secretKeys = [ + "QA_TEST_EMAIL", + "QA_TEST_PASSWORD", + "VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET", + "VERCEL_DONOR_AUTOMATION_BYPASS_SECRET", + "VERCEL_MISSIONARY_AUTOMATION_BYPASS_SECRET", +] as const; + +function secretRepresentations() { + const variants = new Set(); + for (const name of secretKeys) { + const configured = process.env[name]; + if (!configured) continue; + for (const secret of new Set([configured, configured.trim()])) { + if (!secret) continue; + const base64 = Buffer.from(secret, "utf8").toString("base64"); + const base64url = base64.replaceAll("+", "-").replaceAll("/", "_"); + for (const variant of [ + secret, + encodeURI(secret), + encodeURIComponent(secret), + new URLSearchParams({ value: secret }).toString().slice(6), + base64, + base64.replace(/=+$/u, ""), + base64url, + base64url.replace(/=+$/u, ""), + ]) + variants.add(variant); + } + } + return [...variants].sort((left, right) => right.length - left.length); +} + +function redact(value: unknown, limit = 200): string | null { + if (typeof value !== "string") return null; + const patterns = secretRepresentations().map((variant) => + variant + .replace(/[|\\{}()[\]^$+*?.]/gu, "\\$&") + .replace(/%[0-9a-f]{2}/giu, (escape) => + escape.replace( + /[a-f]/giu, + (letter) => "[" + letter.toLowerCase() + letter.toUpperCase() + "]", + ), + ), + ); + // One replacement pass avoids rescanning the replacement marker. This is a + // bounded set of known representations, not arbitrary obfuscation detection. + const redacted = patterns.length + ? value.replace(new RegExp(patterns.join("|"), "gu"), "[redacted]") + : value; + return redacted.slice(0, limit); +} + +function canonicalPath(path: string): string { + let current = resolve(path); + const missing: string[] = []; + for (;;) { + try { + return resolve(realpathSync(current), ...missing); + } catch (error) { + // Follow the nearest existing parent for new run directories. A dangling + // symlink or inaccessible/non-directory parent is not a missing suffix. + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (lstatSync(current, { throwIfNoEntry: false })) throw error; + const parent = dirname(current); + if (parent === current) throw error; + missing.unshift(basename(current)); + current = parent; + } + } +} + +function inside(root: string, path: string) { + const child = relative(root, path); + return ( + child !== "" && + child !== ".." && + !child.startsWith(".." + sep) && + !isAbsolute(child) + ); +} + +export function assertSmokeArtifactDirectories( + report: string, + outputs: string[], +) { + const reportDirectory = canonicalPath(report); + const outputDirectories = [...new Set(outputs.map(canonicalPath))]; + const workspace = canonicalPath(process.cwd()); + if ( + [reportDirectory, ...outputDirectories].some((directory) => { + const existing = lstatSync(directory, { throwIfNoEntry: false }); + return ( + directory === parse(directory).root || + directory === workspace || + inside(directory, workspace) || + Boolean(existing && !existing.isDirectory()) + ); + }) || + outputDirectories.some( + (output) => + output === reportDirectory || + inside(reportDirectory, output) || + inside(output, reportDirectory), + ) + ) { + throw new Error( + "Smoke report and test-output directories must be separate run directories.", + ); + } + return { reportDirectory, outputDirectories }; +} + +function safeEvidence(value: unknown) { + if (!value || typeof value !== "object") return null; + const record = value as Record; + let location = null; + if (typeof record.url === "string") { + try { + const url = new URL(record.url); + location = { origin: redact(url.origin), pathname: redact(url.pathname) }; + } catch { + // Invalid URLs carry no useful origin/path evidence. + } + } + return { + location, + title: redact(record.title), + heading: redact(record.heading, 120), + visiblePasswordInputs: + typeof record.visiblePasswordInputs === "number" + ? record.visiblePasswordInputs + : null, + network: Array.isArray(record.network) + ? record.network.slice(-50).flatMap((entry: unknown) => { + if (!entry || typeof entry !== "object") return []; + const response = entry as Record; + if ( + typeof response.method !== "string" || + ![ + "GET", + "HEAD", + "POST", + "PUT", + "PATCH", + "DELETE", + "OPTIONS", + ].includes(response.method) || + typeof response.status !== "number" || + !Number.isInteger(response.status) || + response.status < 100 || + response.status > 599 || + typeof response.url !== "string" + ) + return []; + try { + const url = new URL(response.url); + return [ + { + method: response.method, + status: response.status, + origin: redact(url.origin), + pathname: redact(url.pathname), + }, + ]; + } catch { + return []; + } + }) + : [], + }; +} + +function escapeHtml(value: string) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +type ResultRecord = { + title: string | null; + project: string | null; + status: TestResult["status"]; + duration: number; + evidence: ReturnType; +}; + +/** This credential-bearing suite publishes allowlisted diagnostics only. */ +export default class DevelopmentSmokeReporter implements Reporter { + private reportDirectory: string; + private outputDirectories: string[] = []; + private results: ResultRecord[] = []; + private evidenceFiles: { directory: string; record: ResultRecord }[] = []; + private prepared = false; + + constructor(options: { outputFolder: string }) { + this.reportDirectory = resolve(options.outputFolder); + } + + onBegin(config: FullConfig, suite: Suite) { + this.outputDirectories = [ + ...new Set(config.projects.map((project) => project.outputDir)), + ]; + const directories = assertSmokeArtifactDirectories( + this.reportDirectory, + this.outputDirectories, + ); + this.reportDirectory = directories.reportDirectory; + this.outputDirectories = directories.outputDirectories; + rmSync(this.reportDirectory, { recursive: true, force: true }); + mkdirSync(this.reportDirectory, { recursive: true }); + this.prepared = true; + console.log(`Running ${suite.allTests().length} smoke tests`); + } + + onTestEnd(test: TestCase, result: TestResult) { + let evidence: ReturnType = null; + for (const attachment of result.attachments) { + let ownedPath: string | undefined; + if (attachment.path) { + try { + const resolved = canonicalPath(attachment.path); + if (this.outputDirectories.some((root) => inside(root, resolved))) + ownedPath = resolved; + } catch { + // Unresolvable paths are not owned evidence sources. + } + } + if (attachment.name === "evidence.json") { + try { + const contents = + attachment.body ?? (ownedPath ? readFileSync(ownedPath) : null); + if (contents) + evidence = safeEvidence(JSON.parse(contents.toString("utf8"))); + } catch { + // Malformed/unavailable attachments do not enter the public bundle. + } + } + } + const record: ResultRecord = { + title: redact(test.title, 500), + project: redact(test.parent.project()?.name), + status: result.status, + duration: result.duration, + evidence, + }; + this.results.push(record); + const projectOutput = test.parent.project()?.outputDir; + const output = projectOutput ? canonicalPath(projectOutput) : undefined; + if (output && this.outputDirectories.includes(output)) { + const id = createHash("sha256") + .update(test.id) + .digest("hex") + .slice(0, 20); + const directory = join(output, `safe-${id}-retry-${result.retry}`); + this.evidenceFiles.push({ directory, record }); + } + console.log(`${record.project}: ${record.status}`); + } + + onEnd(result: FullResult) { + if (!this.prepared) return; + const current = assertSmokeArtifactDirectories( + this.reportDirectory, + this.outputDirectories, + ); + if ( + current.reportDirectory !== this.reportDirectory || + current.outputDirectories.some( + (directory, index) => directory !== this.outputDirectories[index], + ) + ) { + throw new Error("Smoke artifact directories changed during the run."); + } + // Retain only the bounded bundle. This also removes unattached source + // files copied by testInfo.attach(), never arbitrary attachment sources. + for (const output of this.outputDirectories) { + rmSync(output, { recursive: true, force: true }); + mkdirSync(output, { recursive: true }); + } + for (const { directory, record } of this.evidenceFiles) { + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "evidence.json"), + JSON.stringify(record, null, 2) + "\n", + ); + } + const json = JSON.stringify( + { status: result.status, tests: this.results }, + null, + 2, + ); + const sanitizedDirectory = join(this.reportDirectory, "sanitized"); + mkdirSync(sanitizedDirectory, { recursive: true }); + writeFileSync(join(sanitizedDirectory, "results.json"), json + "\n"); + writeFileSync( + join(sanitizedDirectory, "index.html"), + `Smoke diagnostics

Smoke diagnostics

${escapeHtml(json)}
\n`, + ); + } +} diff --git a/tests/unit/admin/eve-preview-build.test.ts b/tests/unit/admin/eve-preview-build.test.ts new file mode 100644 index 000000000..d2e0c6657 --- /dev/null +++ b/tests/unit/admin/eve-preview-build.test.ts @@ -0,0 +1,63 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ withEve: vi.fn((config) => config) })); + +vi.mock("@next/env", () => ({ loadEnvConfig: vi.fn() })); +vi.mock("@payloadcms/next/withPayload", () => ({ + withPayload: (config: unknown) => config, +})); +vi.mock("@sentry/nextjs", () => ({ + withSentryConfig: (config: unknown) => config, +})); +vi.mock( + "../../../apps/admin/node_modules/eve/dist/src/public/next/index.js", + () => ({ + withEve: mocks.withEve, + }), +); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe("admin Eve service builds", () => { + it("uses the target-aware service dispatcher for a hosted preview", async () => { + vi.stubEnv("VERCEL", "1"); + vi.stubEnv("VERCEL_ENV", "preview"); + vi.stubEnv("VERCEL_TARGET_ENV", "preview"); + + await import("../../../apps/admin/next.config"); + + expect(mocks.withEve).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ eveBuildCommand: "bun run build:service" }), + ); + }); + + it.each([ + { VERCEL: "1", VERCEL_ENV: "production" }, + { + VERCEL: "1", + VERCEL_ENV: "preview", + VERCEL_TARGET_ENV: "production", + }, + { VERCEL_ENV: "preview" }, + {}, + ])( + "keeps the same service dispatcher for a later target (%j)", + async (environment) => { + vi.stubEnv("VERCEL", environment.VERCEL); + vi.stubEnv("VERCEL_ENV", environment.VERCEL_ENV); + vi.stubEnv("VERCEL_TARGET_ENV", environment.VERCEL_TARGET_ENV); + vi.stubEnv("CORE_EVE_BUILD_MODE", "artifacts"); + + await import("../../../apps/admin/next.config"); + + expect(mocks.withEve).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ eveBuildCommand: "bun run build:service" }), + ); + }, + ); +}); diff --git a/tests/unit/auth/middleware.test.ts b/tests/unit/auth/middleware.test.ts index 36fae8dcf..5a6109f05 100644 --- a/tests/unit/auth/middleware.test.ts +++ b/tests/unit/auth/middleware.test.ts @@ -6,6 +6,8 @@ import { E2E_AUTH_COOKIE_NAMES, } from "../../../packages/auth/e2e-auth"; +import type { RoleSnapshot } from "../../../packages/auth/permissions"; + type MockCookieToSet = { name: string; value: string; @@ -528,6 +530,136 @@ describe("createAuthMiddleware", () => { }); }); +describe("preview role-gate diagnostics", () => { + const middlewareFor = (snapshot: RoleSnapshot | null) => + createAuthMiddleware({ + protectedRoutePrefixes: ["/crm"], + allowedRoles: ["staff"], + redirectAuthenticatedTo: "/crm", + unauthorizedRedirectTo: "/no-access", + resolveUserRole: async () => snapshot, + }); + + beforeEach(() => { + mockNoConfig(); + mockConfigWithUser("private-user"); + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("E2E_AUTH_BYPASS", "false"); + vi.stubEnv("VERCEL_ENV", "preview"); + vi.stubEnv("VERCEL_TARGET_ENV", "preview"); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + mockNoConfig(); + }); + + it.each(["/crm", "/login"])( + "identifies role denial on %s while preserving redirect and refreshed cookies", + async (pathname) => { + supabaseCookiesToSetRef.cookies = [ + { + name: "sb-access-token", + value: "private-refresh-cookie", + options: { path: "/", httpOnly: true, sameSite: "lax" }, + }, + ]; + const middleware = middlewareFor({ + profileRole: "donor", + memberships: [], + }); + + const response = await middleware( + createRequest(pathname + "?private=secret-query"), + ); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toBe( + "https://example.org/no-access", + ); + expect(response.cookies.get("sb-access-token")).toMatchObject({ + value: "private-refresh-cookie", + path: "/", + httpOnly: true, + sameSite: "lax", + }); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "role_gate", + outcome: "role_denied", + code: null, + }); + }, + ); + + it.each(["/crm", "/login"])( + "keeps %s denied with refreshed cookies when the diagnostic sink throws", + async (pathname) => { + vi.mocked(console.warn).mockImplementation(() => { + throw new Error("private sink error"); + }); + supabaseCookiesToSetRef.cookies = [ + { + name: "sb-access-token", + value: "private-refresh-cookie", + options: { path: "/", httpOnly: true }, + }, + ]; + const middleware = middlewareFor({ + profileRole: "donor", + memberships: [], + }); + const response = await middleware(createRequest(pathname)); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toBe( + "https://example.org/no-access", + ); + expect(response.cookies.get("sb-access-token")?.value).toBe( + "private-refresh-cookie", + ); + expect(console.warn).toHaveBeenCalledTimes(1); + }, + ); + + it.each(["/crm", "/login"])( + "keeps %s denied but silent when production contradicts a preview target", + async (pathname) => { + vi.stubEnv("VERCEL_ENV", "production"); + const middleware = middlewareFor({ + profileRole: "donor", + memberships: [], + }); + const response = await middleware(createRequest(pathname)); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toBe( + "https://example.org/no-access", + ); + expect(console.warn).not.toHaveBeenCalled(); + }, + ); + + it.each(["/crm", "/login"])( + "preserves allowed and unresolved behavior on %s without a role-denial event", + async (pathname) => { + const allowed = await middlewareFor({ + profileRole: "staff", + memberships: [], + })(createRequest(pathname)); + expect(allowed.status).toBe(pathname === "/login" ? 307 : 200); + expect(allowed.headers.get("location")).toBe( + pathname === "/login" ? "https://example.org/crm" : null, + ); + const unresolved = await middlewareFor(null)(createRequest(pathname)); + expect(unresolved.status).toBe(307); + expect(unresolved.headers.get("location")).toBe( + "https://example.org/no-access", + ); + expect(console.warn).not.toHaveBeenCalled(); + }, + ); +}); + describe("edge role enforcement", () => { beforeEach(() => { process.env.E2E_AUTH_BYPASS = "false"; diff --git a/tests/unit/auth/resolve-user-role.test.ts b/tests/unit/auth/resolve-user-role.test.ts index f1867c941..293a6a9e9 100644 --- a/tests/unit/auth/resolve-user-role.test.ts +++ b/tests/unit/auth/resolve-user-role.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveUserRoleFromDatabase } from "../../../packages/auth/resolve-user-role"; @@ -28,12 +28,16 @@ function fakeSupabase({ memberships = [], throwOn, returnErrorOn, + queryError, + thrownError, rpcCalls, }: { profile?: ProfileRow; memberships?: MembershipRow[]; throwOn?: "profiles" | "memberships"; returnErrorOn?: "profiles" | "memberships"; + queryError?: unknown; + thrownError?: unknown; rpcCalls?: RpcCall[]; }) { return { @@ -46,13 +50,13 @@ function fakeSupabase({ eq: () => ({ maybeSingle: async () => { if (throwOn === "profiles") { - throw new Error("profiles unavailable"); + throw thrownError ?? new Error("profiles unavailable"); } return { data: returnErrorOn === "profiles" ? null : profile, error: returnErrorOn === "profiles" - ? new Error("profiles unavailable") + ? (queryError ?? new Error("profiles unavailable")) : null, }; }, @@ -69,14 +73,14 @@ function fakeSupabase({ rpcCalls?.push({ fn, args }); if (throwOn === "memberships") { - throw new Error("memberships unavailable"); + throw thrownError ?? new Error("memberships unavailable"); } return { data: returnErrorOn === "memberships" ? null : memberships, error: returnErrorOn === "memberships" - ? new Error("memberships unavailable") + ? (queryError ?? new Error("memberships unavailable")) : null, }; }, @@ -203,3 +207,265 @@ describe("resolveUserRoleFromDatabase", () => { expect(snapshot).toBeNull(); }); }); + +describe("preview role-resolution diagnostics", () => { + beforeEach(() => { + // Hosted previews are production builds, not protected production targets. + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("VERCEL_ENV", "preview"); + vi.stubEnv("VERCEL_TARGET_ENV", "preview"); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("distinguishes no visible profile from a query failure while denying both", async () => { + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ profile: null }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenLastCalledWith({ + event: "auth_access_diagnostic", + stage: "profile_read", + outcome: "no_visible_profile", + code: null, + }); + + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ + returnErrorOn: "profiles", + queryError: { code: "42501", message: "private query detail" }, + }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenLastCalledWith({ + event: "auth_access_diagnostic", + stage: "profile_read", + outcome: "query_failed", + code: "42501", + }); + expect(console.warn).toHaveBeenCalledTimes(2); + }); + + it("identifies a failed membership RPC without changing the null result", async () => { + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ + profile: { tenant_id: "private-tenant", role: "donor" }, + returnErrorOn: "memberships", + queryError: { code: "PGRST202", details: "private RPC detail" }, + }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "membership_read", + outcome: "query_failed", + code: "PGRST202", + }); + }); + + it.each(["profiles", "memberships"] as const)( + "keeps a thrown %s failure closed with fixed exception metadata", + async (throwOn) => { + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ + profile: { tenant_id: "private-tenant", role: "donor" }, + throwOn, + thrownError: new Error("private thrown detail"), + }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "resolver", + outcome: "exception", + code: "other", + }); + }, + ); + + it.each([ + ["production", "preview"], + ["preview", "production"], + ["preview", "core-development"], + ["preview", "staging"], + [" Production ", "preview"], + ["development", ""], + ["", ""], + ])( + "stays silent and denied for VERCEL_ENV=%s and target=%s", + async (env, target) => { + vi.stubEnv("VERCEL_ENV", env); + vi.stubEnv("VERCEL_TARGET_ENV", target); + const readCode = vi.fn(() => "42501"); + const queryError = Object.defineProperty({}, "code", { get: readCode }); + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ returnErrorOn: "profiles", queryError }), + }), + ).toBeNull(); + expect(readCode).not.toHaveBeenCalled(); + expect(console.warn).not.toHaveBeenCalled(); + }, + ); + + it("uses normalized preview classification and stays silent for a valid snapshot", async () => { + vi.stubEnv("VERCEL_ENV", " Preview "); + vi.stubEnv("VERCEL_TARGET_ENV", ""); + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ + profile: { tenant_id: "private-tenant", role: "donor" }, + }), + }), + ).toEqual({ profileRole: "donor", memberships: [] }); + expect(console.warn).not.toHaveBeenCalled(); + + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ profile: null }), + }); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "profile_read", + outcome: "no_visible_profile", + code: null, + }); + }); + + it.each([ + "42501", + "42P01", + "42883", + "PGRST106", + "PGRST116", + "PGRST202", + "PGRST301", + ])("retains only the allowlisted query code %s", async (code) => { + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ + returnErrorOn: "profiles", + queryError: { code, message: "private error" }, + }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "profile_read", + outcome: "query_failed", + code, + }); + }); + + it("excludes raw and encoded identities, error fields and unknown codes", async () => { + const secret = "dummy+password@example.test"; + const variants = [ + secret, + encodeURIComponent(secret), + Buffer.from(secret).toString("base64"), + ]; + const serialize = vi.fn(() => secret); + for (const variant of variants) { + expect( + await resolveUserRoleFromDatabase({ + userId: variant, + supabase: fakeSupabase({ + profile: { tenant_id: variant, role: "donor" }, + returnErrorOn: "memberships", + queryError: { + code: variant, + message: variant, + details: variant, + hint: variant, + cause: { userId: variant, tenantId: variant }, + stage: variant, + outcome: variant, + toJSON: serialize, + toString: serialize, + }, + }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenLastCalledWith({ + event: "auth_access_diagnostic", + stage: "membership_read", + outcome: "query_failed", + code: "other", + }); + } + const output = JSON.stringify(vi.mocked(console.warn).mock.calls); + for (const variant of variants) expect(output).not.toContain(variant); + expect(serialize).not.toHaveBeenCalled(); + }); + + it.each(["getter", "revoked-proxy", "getter-and-sink"])( + "keeps an unreadable %s code closed with an other-code event", + async (kind) => { + if (kind === "getter-and-sink") { + vi.mocked(console.warn).mockImplementation(() => { + throw new Error("private sink detail"); + }); + } + const queryError = kind.startsWith("getter") + ? Object.defineProperty({}, "code", { + get() { + throw new Error("private getter detail"); + }, + }) + : (() => { + const value = Proxy.revocable({}, {}); + value.revoke(); + return value.proxy; + })(); + expect( + await resolveUserRoleFromDatabase({ + userId: "private-user", + supabase: fakeSupabase({ returnErrorOn: "profiles", queryError }), + }), + ).toBeNull(); + expect(console.warn).toHaveBeenCalledExactlyOnceWith({ + event: "auth_access_diagnostic", + stage: "profile_read", + outcome: "query_failed", + code: "other", + }); + }, + ); + + it.each(["profile", "membership", "exception"])( + "preserves the denied %s result when logging throws", + async (kind) => { + vi.mocked(console.warn).mockImplementation(() => { + throw new Error("private sink detail"); + }); + const supabase = + kind === "profile" + ? fakeSupabase({ profile: null }) + : kind === "membership" + ? fakeSupabase({ + profile: { tenant_id: "private-tenant", role: "donor" }, + returnErrorOn: "memberships", + }) + : fakeSupabase({ throwOn: "profiles" }); + await expect( + resolveUserRoleFromDatabase({ userId: "private-user", supabase }), + ).resolves.toBeNull(); + expect(console.warn).toHaveBeenCalledTimes(1); + }, + ); +}); diff --git a/tests/unit/playwright-development-smoke-output.test.ts b/tests/unit/playwright-development-smoke-output.test.ts new file mode 100644 index 000000000..658291e43 --- /dev/null +++ b/tests/unit/playwright-development-smoke-output.test.ts @@ -0,0 +1,348 @@ +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { extname, join, resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; + +const require = createRequire(import.meta.url); +const playwrightModule = require.resolve("@playwright/test"); +const playwrightCli = require.resolve("@playwright/test/cli"); +const smokeConfig = resolve("playwright.development-smoke.config.ts"); +const sentinels = { + email: "artifact+canary@example.test", + password: "dummy+pw@example.test", + bypass: "by?+ /x", +}; + +function createFailingFixture(directory: string, encoded = false) { + writeFileSync(join(directory, "package.json"), '{"private":true}\n'); + writeFileSync( + join(directory, "playwright.config.ts"), + `import config from ${JSON.stringify(smokeConfig)}; +export default { ...config, testDir: ${JSON.stringify(directory)} };`, + ); + writeFileSync(join(directory, "outside-evidence.txt"), sentinels.password); + writeFileSync( + join(directory, "admin.artifact-path.spec.ts"), + `import playwright from ${JSON.stringify(playwrightModule)}; +import { writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { once } from "node:events"; +const { test, expect } = playwright; +const encoded = ${encoded}; +test("preserves a failed auth check with safe evidence", async ({ request }, info) => { + const server = createServer((req, res) => res.end('{"ok":false}')); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + await request.post("http://127.0.0.1:" + server.address().port + "/auth", { + data: { email: process.env.QA_TEST_EMAIL, password: process.env.QA_TEST_PASSWORD }, + }); + const raw = info.outputPath("raw-error.txt"); + await writeFile(raw, process.env.QA_TEST_PASSWORD); + await info.attach("raw-error", { path: raw, contentType: "text/plain" }); + info.attachments.push({ name: "outside", path: process.env.OUTSIDE_EVIDENCE_PATH, contentType: "text/plain" }); + await info.attach("evidence.json", { + body: JSON.stringify({ + url: encoded + ? "https://preview.example.test/no-access/" + encodeURIComponent(process.env.QA_TEST_PASSWORD) + : "https://preview.example.test/no-access?token=" + process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET + "#private-fragment", + title: encoded + ? "Access: " + encodeURIComponent(process.env.QA_TEST_PASSWORD) + " | " + encodeURIComponent(process.env.QA_TEST_EMAIL) + " | " + encodeURI(process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET) + : "Access: " + process.env.QA_TEST_EMAIL, + heading: encoded + ? "Denied: " + Buffer.from(process.env.QA_TEST_PASSWORD).toString("base64") + " | " + Buffer.from(process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET).toString("base64url") + : "Denied: " + process.env.QA_TEST_PASSWORD, + visiblePasswordInputs: 1, + ignoredPrivateField: process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET, + network: encoded ? [ + encodeURIComponent(process.env.QA_TEST_PASSWORD).replace(/%[a-f0-9]{2}/gi, value => value.toLowerCase()), + new URLSearchParams({value:process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET}).toString().slice(6), + Buffer.from(process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET).toString("base64"), + Buffer.from(process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET).toString("base64").replace(/=+$/, ""), + Buffer.from(process.env.VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET).toString("base64").replaceAll("+", "-").replaceAll("/", "_"), + ].map(value => ({method:"POST",status:403,url:"https://auth.example.test/reset/"+value})) : [{ + method: "GET", status: 406, + url: "https://auth.example.test/rest/v1/profiles?email=" + process.env.QA_TEST_EMAIL + "#private-fragment", + headers: { authorization: process.env.QA_TEST_PASSWORD }, + }], + }), + contentType: "application/json", + }); + await test.step("API login " + process.env.QA_TEST_EMAIL, async () => { + expect(process.env.QA_TEST_PASSWORD).toBe("deliberate failure"); + }); + } finally { + server.close(); + } +});`, + ); +} + +function runFixture( + directory: string, + overrides: { report?: string; output?: string } = {}, + startsTest = true, +) { + const environment: Record = { + CI: "1", + FORCE_COLOR: "0", + QA_TEST_EMAIL: sentinels.email, + QA_TEST_PASSWORD: sentinels.password, + VERCEL_ADMIN_AUTOMATION_BYPASS_SECRET: sentinels.bypass, + OUTSIDE_EVIDENCE_PATH: join(directory, "outside-evidence.txt"), + }; + // Never inherit real QA or provider credentials into the subprocess. + for (const key of [ + "PATH", + "HOME", + "USERPROFILE", + "SystemRoot", + "TEMP", + "TMP", + ]) { + const value = process.env[key]; + if (value) environment[key] = value; + } + if (overrides.report !== undefined) + environment.PLAYWRIGHT_REPORT_DIR = overrides.report; + if (overrides.output !== undefined) + environment.PLAYWRIGHT_OUTPUT_DIR = overrides.output; + const result = spawnSync( + process.execPath, + [ + playwrightCli, + "test", + "--config", + join(directory, "playwright.config.ts"), + "--project", + "development-admin", + ], + { cwd: directory, env: environment, encoding: "utf8", timeout: 30_000 }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stdout + result.stderr).toBe(1); + expect(result.stdout + result.stderr).toContain( + startsTest + ? "Running 1 smoke tests" + : "Smoke report and test-output directories must be separate run directories.", + ); + for (const secret of Object.values(sentinels)) { + expect(result.stdout + result.stderr).not.toContain(secret); + } + expect(readFileSync(join(directory, "outside-evidence.txt"), "utf8")).toBe( + sentinels.password, + ); +} + +function files(directory: string): string[] { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + return entry.isDirectory() ? files(path) : [path]; + }); +} + +function expectSafeArtifacts( + directory: string, + report: string, + output: string, + encoded = false, +) { + const reportDirectory = resolve(directory, report); + const outputDirectory = resolve(directory, output); + expect(existsSync(join(reportDirectory, "sanitized/index.html"))).toBe(true); + const paths = [...files(reportDirectory), ...files(outputDirectory)]; + // Reject opaque binary/compressed attachments instead of trusting a raw grep. + for (const path of paths) { + expect([".html", ".json", ".md"]).toContain(extname(path)); + const contents = readFileSync(path, "utf8"); + expect(contents).not.toMatch(/ ({ + method: "POST", + status: 403, + origin: "https://auth.example.test", + pathname: "/reset/[redacted]", + })) + : [ + { + method: "GET", + status: 406, + origin: "https://auth.example.test", + pathname: "/rest/v1/profiles", + }, + ], + }, + }); + expect(bundle.tests[0].duration).toBeGreaterThan(0); + const evidence = paths.filter((path) => path.endsWith("evidence.json")); + expect(evidence).toHaveLength(1); + expect(JSON.parse(readFileSync(evidence[0]!, "utf8"))).toEqual( + bundle.tests[0], + ); +} + +describe("development smoke artifact output", () => { + it("redacts URL and base64 canaries placed in retained fields", () => { + const directory = mkdtempSync(join(tmpdir(), "core-smoke-encoded-")); + try { + createFailingFixture(directory, true); + runFixture(directory); + expectSafeArtifacts( + directory, + "playwright-report/development-smoke", + "test-results", + true, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 60_000); + + it("leaves no uploadable bundle when reporter completion fails", () => { + const directory = mkdtempSync( + join(tmpdir(), "core-smoke-reporter-failure-"), + ); + try { + createFailingFixture(directory); + writeFileSync( + join(directory, "crashing-reporter.ts"), + `import SafeReporter from ${JSON.stringify(resolve("tests/e2e/development-smoke/safe-reporter.ts"))}; +export default class extends SafeReporter { onEnd() { throw new Error("Controlled reporter completion failure"); } }`, + ); + writeFileSync( + join(directory, "playwright.config.ts"), + `import config from ${JSON.stringify(smokeConfig)}; +export default {...config, testDir:${JSON.stringify(directory)}, reporter:[[${JSON.stringify(join(directory, "crashing-reporter.ts"))},{outputFolder:"playwright-report/pr-preview-smoke-admin"}]]};`, + ); + runFixture(directory, { + report: "playwright-report/pr-preview-smoke-admin", + output: "test-results/pr-preview-smoke-admin", + }); + expect( + files(join(directory, "test-results")).some((path) => + readFileSync(path, "utf8").includes(sentinels.password), + ), + ).toBe(true); + expect( + existsSync( + join( + directory, + "playwright-report/pr-preview-smoke-admin/sanitized/index.html", + ), + ), + ).toBe(false); + expect( + existsSync( + join( + directory, + "playwright-report/pr-preview-smoke-admin/sanitized/results.json", + ), + ), + ).toBe(false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 60_000); + + it.each([ + { report: ".", output: "test-results" }, + { report: "playwright-report/guard", output: "." }, + ])( + "rejects workspace output roots before startup cleanup: %j", + (override) => { + const directory = mkdtempSync(join(tmpdir(), "core-smoke-root-guard-")); + try { + createFailingFixture(directory); + const config = readFileSync( + join(directory, "playwright.config.ts"), + "utf8", + ); + runFixture(directory, override, false); + expect( + readFileSync(join(directory, "playwright.config.ts"), "utf8"), + ).toBe(config); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, + 60_000, + ); + + it("retains safe failures from separate workflow-selected surface paths", () => { + const directory = mkdtempSync(join(tmpdir(), "core-preview-artifacts-")); + try { + createFailingFixture(directory); + for (const surface of ["admin", "donor"]) { + runFixture(directory, { + report: `playwright-report/pr-preview-smoke-${surface}`, + output: `test-results/pr-preview-smoke-${surface}`, + }); + } + for (const surface of ["admin", "donor"]) { + expectSafeArtifacts( + directory, + `playwright-report/pr-preview-smoke-${surface}`, + `test-results/pr-preview-smoke-${surface}`, + ); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 60_000); + + it("keeps safe local defaults when overrides are blank", () => { + const directory = mkdtempSync( + join(tmpdir(), "core-development-artifacts-"), + ); + try { + createFailingFixture(directory); + runFixture(directory, { report: " ", output: " " }); + expectSafeArtifacts( + directory, + "playwright-report/development-smoke", + "test-results", + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/tests/unit/playwright-development-smoke-paths.test.ts b/tests/unit/playwright-development-smoke-paths.test.ts new file mode 100644 index 000000000..caf54c028 --- /dev/null +++ b/tests/unit/playwright-development-smoke-paths.test.ts @@ -0,0 +1,159 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import SafeReporter, { + assertSmokeArtifactDirectories, +} from "../../tests/e2e/development-smoke/safe-reporter"; + +import type { FullConfig, FullResult, Suite } from "@playwright/test/reporter"; +import type * as Fs from "node:fs"; + +const mutations = vi.hoisted(() => ({ + remove: vi.fn(), + mkdir: vi.fn(), + write: vi.fn(), +})); +vi.mock("node:fs", async (importOriginal) => ({ + ...(await importOriginal()), + rmSync: mutations.remove, + mkdirSync: mutations.mkdir, + writeFileSync: mutations.write, +})); +const realFs = await vi.importActual("node:fs"); + +afterEach(() => vi.restoreAllMocks()); + +function fixture() { + vi.clearAllMocks(); + const root = realFs.mkdtempSync(join(tmpdir(), "core-smoke-alias-guard-")); + const workspace = join(root, "workspace"); + realFs.mkdirSync(workspace); + realFs.writeFileSync( + join(workspace, "sentinel.txt"), + "workspace must survive", + ); + realFs.symlinkSync(root, join(workspace, "alias"), "junction"); + const cwd = vi.spyOn(process, "cwd").mockReturnValue(workspace); + return { + root, + workspace, + close() { + cwd.mockRestore(); + expect(realFs.readFileSync(join(workspace, "sentinel.txt"), "utf8")).toBe( + "workspace must survive", + ); + realFs.rmSync(root, { recursive: true, force: true }); + }, + }; +} + +function runReporter(report: string, output: string) { + const reporter = new SafeReporter({ outputFolder: report }); + reporter.onBegin( + { projects: [{ outputDir: output }] } as FullConfig, + { allTests: () => [] } as unknown as Suite, + ); + reporter.onEnd({ status: "failed" } as FullResult); +} + +describe("smoke artifact canonical path safety", () => { + it.each([ + "output-equals-workspace", + "report-ancestor", + "same-new-target", + "nested-new-report", + "nested-new-output", + ] as const)( + "rejects %s through an existing symlink before requesting removal", + (kind) => { + const setup = fixture(); + try { + const { root, workspace } = setup; + const cases = { + "output-equals-workspace": [ + join(root, "report"), + join(workspace, "alias", "workspace"), + ], + "report-ancestor": [ + join(workspace, "alias"), + join(workspace, "output"), + ], + "same-new-target": [ + join(root, "new-run"), + join(workspace, "alias", "new-run"), + ], + "nested-new-report": [ + join(workspace, "alias", "new-output", "report"), + join(root, "new-output"), + ], + "nested-new-output": [ + join(root, "new-report"), + join(workspace, "alias", "new-report", "output"), + ], + }; + const [report, output] = cases[kind]; + expect(() => runReporter(report!, output!)).toThrow( + "Smoke report and test-output directories must be separate run directories.", + ); + expect(mutations.remove).not.toHaveBeenCalled(); + } finally { + setup.close(); + } + }, + ); + + it("permits separate new run directories through a resolved parent", () => { + const setup = fixture(); + try { + expect( + assertSmokeArtifactDirectories(join(setup.workspace, "new-report"), [ + join(setup.workspace, "alias", "new-output"), + ]), + ).toEqual({ + reportDirectory: join(setup.workspace, "new-report"), + outputDirectories: [join(setup.root, "new-output")], + }); + expect(mutations.remove).not.toHaveBeenCalled(); + } finally { + setup.close(); + } + }); + + it("rejects a dangling symlink instead of treating it as a new parent", () => { + const setup = fixture(); + try { + const dangling = join(setup.workspace, "dangling"); + realFs.symlinkSync(join(setup.root, "missing"), dangling, "junction"); + expect(() => + runReporter(join(setup.workspace, "report"), join(dangling, "output")), + ).toThrow(); + expect(mutations.remove).not.toHaveBeenCalled(); + } finally { + setup.close(); + } + }); + + it("rejects an output rebound to the workspace before final cleanup", () => { + const setup = fixture(); + try { + const output = join(setup.workspace, "new-output"); + const reporter = new SafeReporter({ + outputFolder: join(setup.workspace, "report"), + }); + reporter.onBegin( + { projects: [{ outputDir: output }] } as FullConfig, + { allTests: () => [] } as unknown as Suite, + ); + mutations.remove.mockClear(); + realFs.symlinkSync(setup.workspace, output, "junction"); + expect(() => reporter.onEnd({ status: "failed" } as FullResult)).toThrow( + "Smoke report and test-output directories must be separate run directories.", + ); + expect(mutations.remove).not.toHaveBeenCalled(); + } finally { + setup.close(); + } + }); +}); diff --git a/tests/unit/scripts/ci-build.test.ts b/tests/unit/scripts/ci-build.test.ts index 14131d23a..b4a91e558 100644 --- a/tests/unit/scripts/ci-build.test.ts +++ b/tests/unit/scripts/ci-build.test.ts @@ -13,6 +13,26 @@ import { } from "../../../scripts/verify/ci-build.mjs"; describe("ci-build command planning", () => { + it("marks web dependency and app builds as Eve artifacts only", () => { + const app = { + id: "admin", + filter: "@asym/admin", + cwd: "apps/admin", + nextDir: "apps/admin/.next", + }; + for (const platform of ["linux", "win32"]) { + for (const strict of [false, true]) { + const steps = [ + ...getSharedPackageBuildSteps({ platform, strict, apps: [app] }), + getAppBuildStep(app, { platform, strict }), + ]; + for (const step of steps) { + expect(step.env).toEqual({ CORE_EVE_BUILD_MODE: "artifacts" }); + } + } + } + }); + it("keeps the workspace-link repair wired into CI and postinstall", () => { // CI correctness depends on these two call sites, but neither is reachable // from a unit test - only the repair script's own tests exercise the logic. @@ -100,6 +120,7 @@ describe("ci-build command planning", () => { expect(steps).toContainEqual({ label: "api", + env: { CORE_EVE_BUILD_MODE: "artifacts" }, command: "node", args: [ "scripts/run-with-ci-env.mjs", @@ -131,6 +152,7 @@ describe("ci-build command planning", () => { ).toEqual([ { label: "shared packages", + env: { CORE_EVE_BUILD_MODE: "artifacts" }, command: "node", args: [ "scripts/run-with-ci-env.mjs", @@ -157,6 +179,7 @@ describe("ci-build command planning", () => { ).toEqual([ { label: "shared packages", + env: { CORE_EVE_BUILD_MODE: "artifacts" }, command: "node_modules/.bin/turbo", args: [ "run", @@ -184,6 +207,7 @@ describe("ci-build command planning", () => { ).toEqual({ label: "admin", command: "node", + env: { CORE_EVE_BUILD_MODE: "artifacts" }, args: [ "scripts/run-with-ci-env.mjs", "--", diff --git a/tests/unit/scripts/eve-build-cli.test.ts b/tests/unit/scripts/eve-build-cli.test.ts new file mode 100644 index 000000000..9639a873e --- /dev/null +++ b/tests/unit/scripts/eve-build-cli.test.ts @@ -0,0 +1,135 @@ +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +const require = createRequire(import.meta.url); +const fixtures: string[] = []; + +afterEach(() => { + for (const directory of fixtures.splice(0)) + rmSync(directory, { recursive: true, force: true }); +}); + +function runScript( + script: string, + args: string[], + mode: string, + signals: Record = {}, +) { + const directory = mkdtempSync(path.join(tmpdir(), "core-eve-build-cli-")); + fixtures.push(directory); + mkdirSync(path.join(directory, "scripts")); + copyFileSync( + "packages/eve-runtime/scripts/build.mjs", + path.join(directory, "scripts/build.mjs"), + ); + const pkg = JSON.parse( + readFileSync("packages/eve-runtime/package.json", "utf8"), + ); + writeFileSync( + path.join(directory, "package.json"), + JSON.stringify({ type: "module", scripts: pkg.scripts }), + ); + const sdk = path.join(directory, "node_modules/eve"); + mkdirSync(path.join(sdk, "bin"), { recursive: true }); + writeFileSync( + path.join(sdk, "package.json"), + JSON.stringify({ name: "eve", type: "module" }), + ); + writeFileSync( + path.join(sdk, "bin/eve.js"), + '#!/usr/bin/env node\nimport {writeFileSync} from "node:fs"; writeFileSync(process.env.EVE_BUILD_TEST_OUTPUT, JSON.stringify(process.argv.slice(2)));\n', + { mode: 0o755 }, + ); + mkdirSync(path.join(directory, "node_modules/.bin")); + symlinkSync( + path.join(sdk, "bin/eve.js"), + path.join(directory, "node_modules/.bin/eve"), + ); + mkdirSync(path.join(directory, "node_modules/@asym")); + symlinkSync( + path.resolve(path.dirname(require.resolve("@asym/env/target-env")), ".."), + path.join(directory, "node_modules/@asym/env"), + "dir", + ); + const output = path.join(directory, "sdk-argv.json"); + const result = spawnSync("bun", ["run", script, ...args], { + cwd: directory, + env: { + PATH: process.env.PATH, + CORE_EVE_BUILD_MODE: mode, + EVE_BUILD_TEST_OUTPUT: output, + ...signals, + }, + encoding: "utf8", + timeout: 15_000, + }); + if (result.error) throw result.error; + return { + status: result.status, + stderr: result.stderr, + sdkArgs: existsSync(output) + ? JSON.parse(readFileSync(output, "utf8")) + : null, + }; +} + +describe("named Eve build commands", () => { + it.each([ + { args: ["--service"] }, + { args: ["--service", "--skip-sandbox-prewarm"] }, + ])( + "rejects a forwarded service selector on explicit full builds (%j)", + ({ args }) => { + const result = runScript("build:full", args, "artifacts", { + VERCEL: "1", + VERCEL_ENV: "preview", + VERCEL_TARGET_ENV: "core-development", + }); + expect(result.status, JSON.stringify(result.sdkArgs)).not.toBe(0); + expect(result.sdkArgs).toBeNull(); + }, + ); + + it("keeps explicit full qualification despite inherited artifact mode", () => { + const result = runScript( + "build:full", + ["--profile", "report.json"], + "artifacts", + ); + expect(result.status, result.stderr).toBe(0); + expect(result.sdkArgs).toEqual(["build", "--profile", "report.json"]); + }); + + it("rejects a skip flag on the explicit full command before starting Eve", () => { + const result = runScript( + "build:full", + ["--skip-sandbox-prewarm"], + "artifacts", + ); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "--skip-sandbox-prewarm is only supported in artifacts mode.", + ); + expect(result.sdkArgs).toBeNull(); + }); + + it("keeps the explicit artifact command distinct from inherited full mode", () => { + const result = runScript("build:artifacts", [], "full"); + expect(result.status, result.stderr).toBe(0); + expect(result.sdkArgs).toEqual(["build", "--skip-sandbox-prewarm"]); + }); +}); diff --git a/tests/unit/scripts/eve-build-output-data-boundary.test.ts b/tests/unit/scripts/eve-build-output-data-boundary.test.ts new file mode 100644 index 000000000..faa8b6f64 --- /dev/null +++ b/tests/unit/scripts/eve-build-output-data-boundary.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +function runScanner(files: Record) { + const root = mkdtempSync(path.join(tmpdir(), "core-eve-data-boundary-")); + try { + const scanner = path.join(root, "scripts/verify/data-boundary-check.mjs"); + mkdirSync(path.dirname(scanner), { recursive: true }); + copyFileSync("scripts/verify/data-boundary-check.mjs", scanner); + for (const [file, content] of Object.entries(files)) { + const target = path.join(root, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, content); + } + return spawnSync(process.execPath, [scanner], { + cwd: root, + encoding: "utf8", + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("Eve generated output data-boundary scope", () => { + it.each([ + "apps/admin/.eve/vercel-services/eve/.vercel/output/functions/server.func/index.mjs", + "apps/admin/.vercel/output/functions/server.func/index.mjs", + ])( + "does not classify generated server output as authored app code: %s", + (file) => { + const result = runScanner({ + "apps/admin/app/page.tsx": + "export default function Page() { return null; }", + [file]: + 'import { createClient } from "@supabase/supabase-js";\nconst historicalMarker = "TWENTY_API_KEY";', + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Data access boundary check passed"); + }, + ); + + it.each([ + "apps/admin/app/example.ts", + "apps/admin/.eve/authored.ts", + "apps/admin/.vercel/authored.ts", + "apps/donor/.eve/vercel-services/authored.ts", + ])( + "continues rejecting raw database imports outside the exact output paths: %s", + (file) => { + const result = runScanner({ + [file]: 'import { createClient } from "@supabase/supabase-js";', + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain(file); + expect(result.stderr).toContain( + "Browser Supabase data-boundary violations", + ); + }, + ); + + it("continues rejecting retired CRM references in authored Eve runtime", () => { + const result = runScanner({ + "apps/admin/app/page.tsx": + "export default function Page() { return null; }", + "packages/eve-runtime/src/authored.ts": + 'const forbidden = "TWENTY_API_KEY";', + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("packages/eve-runtime/src/authored.ts"); + expect(result.stderr).toContain("retired Twenty runtime reference"); + }); +}); diff --git a/tests/unit/scripts/eve-build-output-lint.test.ts b/tests/unit/scripts/eve-build-output-lint.test.ts new file mode 100644 index 000000000..aeae5507a --- /dev/null +++ b/tests/unit/scripts/eve-build-output-lint.test.ts @@ -0,0 +1,33 @@ +import path from "node:path"; + +import { ESLint } from "eslint"; +import { describe, expect, it } from "vitest"; + +const root = process.cwd(); +const eslint = new ESLint({ cwd: root }); +const adminEslint = new ESLint({ cwd: path.join(root, "apps/admin") }); +const eveEslint = new ESLint({ cwd: path.join(root, "packages/eve-runtime") }); + +describe("Eve generated build output", () => { + it.each([ + "apps/admin/.eve/vercel-services/eve/.vercel/output/functions/__server.func/index.mjs", + "apps/admin/.vercel/output/functions/generated.func/index.mjs", + ])("does not lint generated deployment code at %s", async (file) => { + expect(await eslint.isPathIgnored(path.resolve(file))).toBe(true); + expect(await adminEslint.isPathIgnored(path.resolve(file))).toBe(true); + }); + + it.each([ + "apps/admin/next.config.ts", + "apps/admin/app/layout.tsx", + "packages/eve-runtime/agent/sandbox.ts", + "packages/eve-runtime/src/governance-boundary.ts", + "packages/eve-runtime/scripts/build.mjs", + ])("continues to lint authored source at %s", async (file) => { + expect(await eslint.isPathIgnored(path.resolve(file))).toBe(false); + const workspaceEslint = file.startsWith("apps/admin/") + ? adminEslint + : eveEslint; + expect(await workspaceEslint.isPathIgnored(path.resolve(file))).toBe(false); + }); +}); diff --git a/tests/unit/scripts/eve-build.test.ts b/tests/unit/scripts/eve-build.test.ts new file mode 100644 index 000000000..ac5d1bf04 --- /dev/null +++ b/tests/unit/scripts/eve-build.test.ts @@ -0,0 +1,167 @@ +import { readFileSync } from "node:fs"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { runEveBuild } from "../../../packages/eve-runtime/scripts/build.mjs"; + +afterEach(() => vi.restoreAllMocks()); + +describe("Eve build dispatch", () => { + it.each([ + { VERCEL: "1", VERCEL_ENV: "production" }, + { VERCEL: "1", VERCEL_ENV: "preview", VERCEL_TARGET_ENV: "production" }, + { VERCEL: "1", VERCEL_ENV: "preview", VERCEL_TARGET_ENV: " Production " }, + { VERCEL: "1", VERCEL_ENV: " Preview ", VERCEL_TARGET_ENV: "production" }, + { VERCEL: "1", VERCEL_ENV: "preview", VERCEL_TARGET_ENV: "development" }, + { + VERCEL: "1", + VERCEL_ENV: "development", + VERCEL_TARGET_ENV: "development", + }, + { VERCEL_ENV: "preview" }, + {}, + ])("keeps service qualification full outside hosted preview (%j)", (env) => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const spawn = vi.fn(() => ({ status: 1 })); + runEveBuild({ + environment: { ...env, CORE_EVE_BUILD_MODE: "artifacts" }, + service: true, + spawn, + }); + expect(spawn).toHaveBeenCalledWith( + process.execPath, + [expect.any(String), "build"], + expect.anything(), + ); + }); + + it.each([undefined, "full"])( + "keeps ordinary/full builds on the required prewarm path (%s)", + (mode) => { + const spawn = vi.fn(() => ({ status: 1 })); + + const status = runEveBuild({ + environment: { CORE_EVE_BUILD_MODE: mode }, + spawn, + }); + + expect(spawn.mock.calls[0]?.[1]).toEqual([ + expect.stringMatching(/[/\\]eve[/\\]bin[/\\]eve\.js$/u), + "build", + ]); + expect(status).toBe(1); + }, + ); + + it("selects the supported skip option only for explicit artifacts", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const spawn = vi.fn(() => ({ status: 0 })); + + expect( + runEveBuild({ + environment: { CORE_EVE_BUILD_MODE: "artifacts" }, + args: ["--profile", "profile.json"], + spawn, + }), + ).toBe(0); + expect(spawn.mock.calls[0]?.[1]).toEqual([ + expect.any(String), + "build", + "--skip-sandbox-prewarm", + "--profile", + "profile.json", + ]); + expect(spawn.mock.calls[0]?.[2]).toEqual( + expect.objectContaining({ shell: false }), + ); + }); + + it.each([ + { VERCEL_ENV: "preview" }, + { VERCEL_ENV: "preview", VERCEL_TARGET_ENV: "preview" }, + { VERCEL_ENV: "preview", VERCEL_TARGET_ENV: "core-development" }, + { VERCEL_ENV: "preview", VERCEL_TARGET_ENV: "staging" }, + { VERCEL_ENV: "preview", VERCEL_TARGET_ENV: " Core-Development " }, + { VERCEL_ENV: " Preview ", VERCEL_TARGET_ENV: "preview" }, + ])( + "compiles a hosted preview service without sandbox prewarming (%j)", + (signals) => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + const spawn = vi.fn(() => ({ status: 0 })); + runEveBuild({ + environment: { + VERCEL: "1", + ...signals, + CORE_EVE_BUILD_MODE: "full", + }, + service: true, + spawn, + }); + expect(spawn).toHaveBeenCalledWith( + process.execPath, + [expect.any(String), "build", "--skip-sandbox-prewarm"], + expect.anything(), + ); + }, + ); + + it.each([false, true])( + "rejects the SDK skip flag in full mode (service=%s)", + (service) => { + const spawn = vi.fn(() => ({ status: 0 })); + expect(() => + runEveBuild({ + environment: { VERCEL: "1", VERCEL_ENV: "production" }, + service, + args: ["--skip-sandbox-prewarm"], + spawn, + }), + ).toThrow("--skip-sandbox-prewarm is only supported in artifacts mode."); + expect(spawn).not.toHaveBeenCalled(); + }, + ); + + it("rejects an unknown mode without starting the SDK", () => { + const spawn = vi.fn(); + expect(() => + runEveBuild({ + environment: { CORE_EVE_BUILD_MODE: "other" }, + spawn, + }), + ).toThrow("CORE_EVE_BUILD_MODE must be full or artifacts."); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("fails when the SDK cannot start or exits without a status", () => { + const error = new Error("spawn failed"); + expect(() => + runEveBuild({ environment: {}, spawn: () => ({ error }) }), + ).toThrow(error); + expect( + runEveBuild({ environment: {}, spawn: () => ({ status: null }) }), + ).toBe(1); + }); + + it("keeps full qualification explicit and distinct from cached artifacts", () => { + const packageJson = JSON.parse( + readFileSync("packages/eve-runtime/package.json", "utf8"), + ); + const turbo = JSON.parse(readFileSync("turbo.json", "utf8")); + const eveTurbo = JSON.parse( + readFileSync("packages/eve-runtime/turbo.json", "utf8"), + ); + + expect(packageJson.scripts["build:full"]).toBe( + "node scripts/build.mjs --full", + ); + expect(packageJson.scripts["build:artifacts"]).toBe( + "node scripts/build.mjs --artifacts", + ); + expect(packageJson.scripts["build:service"]).toBe( + "node scripts/build.mjs --service", + ); + expect(turbo.tasks.build.env).toContain("CORE_EVE_BUILD_MODE"); + expect(eveTurbo.extends).toEqual(["//"]); + expect(eveTurbo.tasks.build.cache).toBe(false); + }); +}); diff --git a/tests/unit/workflows/qa-smoke-preview-deploy.test.ts b/tests/unit/workflows/qa-smoke-preview-deploy.test.ts index 779ac90a2..2e3f7a6ea 100644 --- a/tests/unit/workflows/qa-smoke-preview-deploy.test.ts +++ b/tests/unit/workflows/qa-smoke-preview-deploy.test.ts @@ -105,6 +105,30 @@ describe("qa smoke preview deployment workflow", () => { expect(workflow).not.toContain("bun install --frozen-lockfile"); }); + it("uploads only the bounded sanitized files and fails when they are absent", () => { + const upload = workflow + .split("- name: Upload Playwright smoke artifacts")[1] + ?.split("- name: Comment headless smoke QA result")[0]; + expect(upload).toContain( + "playwright-report/pr-preview-smoke-*/sanitized/index.html", + ); + expect(upload).toContain( + "playwright-report/pr-preview-smoke-*/sanitized/results.json", + ); + expect(upload).toContain("if-no-files-found: error"); + expect(upload).not.toContain("test-results"); + const paths = upload + ?.split("path: |\n")[1] + ?.split("if-no-files-found:")[0] + ?.trim() + .split("\n") + .map((line) => line.trim()); + expect(paths).toEqual([ + "playwright-report/pr-preview-smoke-*/sanitized/index.html", + "playwright-report/pr-preview-smoke-*/sanitized/results.json", + ]); + }); + it("does not use bypass query parameters", () => { expect(workflow).not.toContain("x-vercel-set-bypass-cookie"); expect(workflow).not.toContain("?x-vercel-protection-bypass"); diff --git a/turbo.json b/turbo.json index 13fb9efee..eb84d2ede 100644 --- a/turbo.json +++ b/turbo.json @@ -23,6 +23,7 @@ ], "env": [ "NODE_ENV", + "CORE_EVE_BUILD_MODE", "NEXT_PUBLIC_*", "GOOGLE_SITE_VERIFICATION", "BING_SITE_VERIFICATION",