diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 4402dc219bd7..a7da769dd35b 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -12,8 +12,6 @@ self-hosted-runner: # macOS 26 needs route to Blacksmith cloud, not warp-macos-26-arm64-6x: our # self-hosted minis carry that label, and GitHub prefers a matching # self-hosted runner. See check_no_self_hosted_fleet_runners. - - depot-macos-latest - - depot-macos-14 # Manual E2E canary only. Required CI remains routed through repo variables. - tart-canary # Linux: Blacksmith primary (LINUX_RUNNER), WarpBuild overflow fallback. diff --git a/.github/workflows/perf-activation.yml b/.github/workflows/perf-activation.yml index ff86a246245d..011d9f5edc3f 100644 --- a/.github/workflows/perf-activation.yml +++ b/.github/workflows/perf-activation.yml @@ -9,7 +9,7 @@ on: required: false default: "" runner: - description: macOS runner (auto follows MACOS_RUNNER_15, default WarpBuild; pick blacksmith-/depot-* to override) + description: macOS runner (auto follows MACOS_RUNNER_15; pick a supported runner to override) required: false default: auto type: choice @@ -19,8 +19,6 @@ on: - blacksmith-6vcpu-macos-26 - blacksmith-6vcpu-macos-latest - warp-macos-15-arm64-6x - - depot-macos-latest - - depot-macos-14 workspace_count: description: Fixture workspace count required: false @@ -115,25 +113,6 @@ jobs: env: PERF_TAG: perf-${{ github.run_id }}-${{ github.run_attempt }} steps: - - name: Validate Depot runner identity - if: ${{ startsWith(((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner), 'depot-macos-') }} - env: - REQUESTED_RUNNER: ${{ ((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner) }} - RUNNER_CONTEXT_NAME: ${{ runner.name }} - run: | - set -euo pipefail - echo "Requested runner: $REQUESTED_RUNNER" - case "$RUNNER_CONTEXT_NAME" in - depot-*) - echo "Resolved runner matches depot-*? yes" - ;; - *) - echo "Resolved runner matches depot-*? no" - echo "::error::$REQUESTED_RUNNER resolved outside Depot. Remove $REQUESTED_RUNNER from non-Depot self-hosted runners or choose an explicit runner." - exit 1 - ;; - esac - - name: Clear stale git locks (self-hosted reused workspace) shell: bash run: | diff --git a/.github/workflows/reload-build.yml b/.github/workflows/reload-build.yml index 422751a3ac32..4f056d719674 100644 --- a/.github/workflows/reload-build.yml +++ b/.github/workflows/reload-build.yml @@ -32,7 +32,7 @@ on: runner: description: >- macOS runner label to build on. Blacksmith (blacksmith-6vcpu-macos-26), - our self-hosted fleet (cmux-macos-26 / cmux-aws-macos-15), warp, or depot. + our self-hosted fleet (cmux-macos-26 / cmux-aws-macos-15), or Warp. This is the dev-build offload path (reload-cloud), not required CI, so targeting the fleet for a build is intentional. required: false @@ -71,7 +71,8 @@ concurrency: jobs: build: - runs-on: ${{ inputs.runner }} + # Depot is retired for cmux builds. Legacy Depot requests use Blacksmith. + runs-on: ${{ startsWith(inputs.runner, 'depot-') && 'blacksmith-6vcpu-macos-26' || inputs.runner }} timeout-minutes: 60 env: # The ghostty CLI helper zig build is skipped; GhosttyKit comes prebuilt. diff --git a/.github/workflows/test-depot.yml b/.github/workflows/test-depot.yml index 6a4fc374bc6b..5b1c8095e4cf 100644 --- a/.github/workflows/test-depot.yml +++ b/.github/workflows/test-depot.yml @@ -1,11 +1,8 @@ -name: Run tests on Depot +name: Run macOS tests on: workflow_call: inputs: - runner: - type: string - default: "" ref: type: string default: "" @@ -26,11 +23,6 @@ on: default: "120" workflow_dispatch: inputs: - runner: - description: Optional hosted macOS runner label - required: false - default: "" - type: string ref: description: Branch or SHA to test required: false @@ -63,7 +55,9 @@ env: jobs: tests: - runs-on: ${{ inputs.runner || vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} + # Depot is retired for cmux CI. Keep this reusable test lane on the + # repository's normal macOS runner and do not expose a provider override. + runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: ${{ inputs.unit_test_suites != '' && 35 || 20 }} steps: - name: Clear stale git locks (self-hosted reused workspace) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 2990f5fd3562..921cfc11c4e8 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -35,8 +35,6 @@ on: - blacksmith-6vcpu-macos-26 - blacksmith-6vcpu-macos-latest - warp-macos-15-arm64-6x - - depot-macos-latest - - depot-macos-14 - tart-canary - tart-dual - tart-small @@ -54,25 +52,6 @@ jobs: SWIFTPM_MIRROR_CONFIG: ${{ github.workspace }}/config/swiftpm/mirrors.json CMUX_CI_MAX_MACOS_SDK_MAJOR: "26" steps: - - name: Validate Depot runner identity - if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner, 'depot-macos-') }} - env: - REQUESTED_RUNNER: ${{ (!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner }} - RUNNER_CONTEXT_NAME: ${{ runner.name }} - run: | - set -euo pipefail - echo "Requested runner: $REQUESTED_RUNNER" - case "$RUNNER_CONTEXT_NAME" in - depot-*) - echo "Resolved runner matches depot-*? yes" - ;; - *) - echo "Resolved runner matches depot-*? no" - echo "::error::$REQUESTED_RUNNER resolved outside Depot. Remove $REQUESTED_RUNNER from non-Depot self-hosted runners or choose an explicit runner." - exit 1 - ;; - esac - - name: Validate Tart canary identity if: ${{ startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner, 'tart-') }} env: diff --git a/scripts/install-zig-ci.sh b/scripts/install-zig-ci.sh index 661a2951a0e2..06e84eccf39e 100755 --- a/scripts/install-zig-ci.sh +++ b/scripts/install-zig-ci.sh @@ -15,7 +15,7 @@ ZIG_SYSTEM_PREFIX="${ZIG_SYSTEM_PREFIX:-/usr/local}" ZIG_SYSTEM_PREFIX="${ZIG_SYSTEM_PREFIX%/}" ZIG_DOWNLOAD_ATTEMPTS="${ZIG_DOWNLOAD_ATTEMPTS:-2}" ZIG_DOWNLOAD_RETRY_DELAY="${ZIG_DOWNLOAD_RETRY_DELAY:-10}" -# Keep the default short enough for the 20-minute Depot job even when a job +# Keep the default short enough for the standard hosted macOS job even when a job # invokes this installer twice. Dedicated release jobs can set a larger value, # up to ZIG_DOWNLOAD_BUDGET_MAX_SECONDS, when their job timeout allows it. ZIG_DOWNLOAD_BUDGET_SECONDS="${ZIG_DOWNLOAD_BUDGET_SECONDS:-480}" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 4fccc3115621..c6ddac317abd 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -1331,7 +1331,7 @@ def test_perf_activation_workflow_keeps_required_status_while_gating_benchmark() assert "needs: activation_changes" in benchmark assert "if: ${{ needs.activation_changes.outputs.macos == 'true' }}" in benchmark # The benchmark routes through MACOS_RUNNER_15 (Blacksmith) for all events, - # including PRs. Depot remains only as a manual workflow_dispatch override. + # including PRs. Manual runner overrides stay outside required CI. assert "vars.MACOS_RUNNER_15" in benchmark assert " - activation_changes" in sentinel diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 0491a95d55e5..a9a697135f91 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -139,13 +139,6 @@ check_e2e_runner_fallbacks() { exit 1 fi - for label in depot-macos-latest depot-macos-14; do - if ! grep -Eq "^[[:space:]]+- ${label}$" "$E2E_FILE"; then - echo "FAIL: test-e2e.yml must expose runner option ${label}" - exit 1 - fi - done - if ! awk ' /^ runner:$/ { in_runner=1; next } in_runner && /^ [A-Za-z0-9_-]+:/ { in_runner=0; in_options=0 } @@ -160,16 +153,6 @@ check_e2e_runner_fallbacks() { exit 1 fi - if ! grep -Fq 'RUNNER_CONTEXT_NAME: ${{ runner.name }}' "$E2E_FILE"; then - echo "FAIL: test-e2e.yml must inspect the actual runner name for Depot runs" - exit 1 - fi - - if ! grep -Fq "startsWith((!inputs.runner || inputs.runner == 'auto') && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || inputs.runner, 'depot-macos-')" "$E2E_FILE"; then - echo "FAIL: test-e2e.yml must validate all Depot macOS runner choices" - exit 1 - fi - if ! awk ' /^[[:space:]]*- name: Validate Tart canary identity$/ { in_tart_step=1; next } in_tart_step && /^ - / { in_tart_step=0; in_runner_reject=0; in_marker_reject=0 } @@ -191,33 +174,12 @@ check_e2e_runner_fallbacks() { exit 1 fi - if ! awk ' - /^[[:space:]]*\*\)$/ { - in_reject = 1 - saw_error = 0 - saw_exit = 0 - next - } - in_reject && /echo "::error::\$REQUESTED_RUNNER resolved outside Depot/ { saw_error = 1 } - in_reject && /^[[:space:]]*exit 1$/ { saw_exit = 1 } - in_reject && /^[[:space:]]*;;$/ { - if (saw_error && saw_exit) { - found = 1 - } - in_reject = 0 - } - END { exit(found ? 0 : 1) } - ' "$E2E_FILE"; then - echo "FAIL: test-e2e.yml must fail fast and explain runner label misrouting clearly" - exit 1 - fi - if grep -Eq "^[[:space:]]*continue-on-error:" "$E2E_FILE"; then echo "FAIL: test-e2e.yml must not mask E2E setup or test failures with continue-on-error" exit 1 fi - echo "PASS: test-e2e.yml exposes Depot and Tart runner choices, identity guards, and duplicate-queue cancellation" + echo "PASS: test-e2e.yml exposes supported Tart runner choices and duplicate-queue cancellation" } check_ios_tart_canary() { @@ -1152,12 +1114,11 @@ check_no_self_hosted_fleet_runners() { # changes and a physical host label cannot bypass the isolated VM pool. # Allowed macOS labels (none carried by any fleet runner): # blacksmith-{6,12}vcpu-macos-{15,26,latest}, warp-macos-15-arm64-6x, - # depot-macos-{latest,14}. # NOTE: reload-build.yml is the dev-build offload path (workflow_dispatch, # not required CI) and intentionally targets the fleet via a free-form input; # this guard only inspects runner-selection lines, not its input description. local fleet='macos-26|warp-macos-26-arm64-6x|cmux-aws-macos|cmux-macos|cmux-local-macos|macfleet|tart-[a-z0-9-]+|(^|[^a-z0-9-])mac4([^a-z0-9]|$)|(^|[^a-z0-9-])mac-mini([^a-z0-9]|$)|slot-[0-9]|xcode-[0-9]+-[0-9]|(^|[^a-z0-9-])cmux([^a-z0-9-]|$)' - local allowed='blacksmith-(6|12)vcpu-macos-(15|26|latest)|warp-macos-15-arm64-6x|depot-macos-(latest|14)' + local allowed='blacksmith-(6|12)vcpu-macos-(15|26|latest)|warp-macos-15-arm64-6x' # Bare self-hosted/macOS/ARM64 targeting (inline array or multi-line list). # Case-sensitive: GitHub's auto labels are `macOS`/`ARM64`, distinct from the @@ -1181,7 +1142,7 @@ check_no_self_hosted_fleet_runners() { for probe in "runs-on: \${{ vars.X || 'blacksmith-6vcpu-macos-26' }}" \ "runs-on: \${{ vars.X || 'blacksmith-12vcpu-macos-26' }}" \ "runs-on: \${{ vars.MACOS_RUNNER_15 || 'warp-macos-15-arm64-6x' }}" \ - '- warp-macos-15-arm64-6x' '- depot-macos-latest' '- blacksmith-6vcpu-macos-15' \ + '- warp-macos-15-arm64-6x' '- blacksmith-6vcpu-macos-15' \ '- blacksmith-4vcpu-ubuntu-2404'; do if printf '%s\n' "$probe" | sed -E "s/($allowed)//g" | grep -Eq "($forbidden)"; then echo "FAIL: fleet-runner guard self-test false-positived a cloud label: $probe" @@ -1279,7 +1240,7 @@ check_macos_runner "$GHOSTTYKIT_FILE" "build-ghosttykit" # ci-macos-compat.yml (matrix.os routed through the MACOS_RUNNER_* repo vars) check_macos_runner "$COMPAT_FILE" "compat-tests" -# test-e2e.yml is manual, so keep the Depot GUI runner choices but cancel +# test-e2e.yml is manual, so keep the supported GUI runner choices but cancel # duplicate queued runs for the same ref/filter/runner. check_e2e_runner_fallbacks check_ios_tart_canary diff --git a/web/oxlint-complexity-baseline.txt b/web/oxlint-complexity-baseline.txt index 0b4e6ca297dd..cef9700bbd38 100644 --- a/web/oxlint-complexity-baseline.txt +++ b/web/oxlint-complexity-baseline.txt @@ -45,7 +45,6 @@ services/subrouter/accountInput.ts 4938ed3780d6d4a540c37a67a8c6120b03ba988e82db4 services/subrouter/hostedClient.ts 82062f7eeda90670adea4a1619e65dbf2cad2d1e05d02196fc191867b36ff7c4 function `parseCredentialLease` has a complexity of 22. Maximum allowed is 20. services/vms/observability.ts b9d616cc0a88f719e82b2005194d7fbab28058e34613a794738972deade85334 function `captureVmRequestOutcome` has a complexity of 38. Maximum allowed is 20. services/vms/observability.ts d15e28b21477d7ce31b23438b3d7640a95fb8313bbe77825400a6380617eed89 function `captureVmProvisionOutcome` has a complexity of 24. Maximum allowed is 20. -services/vms/providerErrors.ts 5c4e725a638f14da834c9943f2e0bf129c59471a1a8399a83fe73150c9fce20c function `isProviderNotFoundError` has a complexity of 24. Maximum allowed is 20. services/vms/providerErrors.ts bdc837c342072c514341cf5deaf8a0b169fe0ab6daf23399580ea83da8fca688 function `isProviderIdentityNotFoundError` has a complexity of 22. Maximum allowed is 20. services/vms/reaper.ts 05bb1f2917bd7a0c88eda164c98509e4b319efca1d398481080e8e15e3310c0b generator function has a complexity of 27. Maximum allowed is 20. services/vms/reaper.ts d59e84b06bcfd550d4e36fd741283757c08519f7cebaed05e3ade6e0cd04c6ed generator function has a complexity of 21. Maximum allowed is 20. diff --git a/web/services/vms/providerErrors.ts b/web/services/vms/providerErrors.ts index 9714cb85ff48..cc121361e52b 100644 --- a/web/services/vms/providerErrors.ts +++ b/web/services/vms/providerErrors.ts @@ -28,24 +28,17 @@ function hasProviderMissingMessage( ); } -export function isProviderNotFoundError(err: unknown): boolean { - if (!err || typeof err !== "object") return false; - const candidate = err as { - code?: string | number; - name?: string; - status?: number; - statusCode?: number; - response?: { status?: number; data?: unknown }; - message?: string; - cause?: unknown; - }; - const status = - candidate.status ?? - candidate.statusCode ?? - candidate.response?.status ?? - undefined; - if (status === 404) return true; +type ProviderFailure = { + code?: string | number; + name?: string; + status?: number; + statusCode?: number; + response?: { status?: number; data?: unknown }; + message?: string; + cause?: unknown; +}; +function hasLegacyProviderNotFoundDetail(candidate: ProviderFailure): boolean { const code = String(candidate.code ?? candidate.name ?? "").toLowerCase(); if ( code === "not_found" || @@ -69,10 +62,33 @@ export function isProviderNotFoundError(err: unknown): boolean { return true; } - if (candidate.cause) return isProviderNotFoundError(candidate.cause); return false; } +function httpStatus(candidate: ProviderFailure): number | undefined { + const candidates = [candidate.status, candidate.statusCode, candidate.response?.status]; + return candidates.find((status): status is number => + typeof status === "number" && status >= 400 && status <= 599, + ); +} + +export function isProviderNotFoundError(err: unknown): boolean { + const seen = new Set(); + let legacyNotFound = false; + let current = err; + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + const candidate = current as ProviderFailure; + const status = httpStatus(candidate); + // The concrete HTTP failure wins over wrapper/code/message heuristics. + // A 502 mentioning a missing VM must never mark the machine destroyed. + if (status !== undefined) return status === 404; + legacyNotFound ||= hasLegacyProviderNotFoundDetail(candidate); + current = candidate.cause; + } + return legacyNotFound; +} + export function isProviderIdentityNotFoundError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const candidate = err as { diff --git a/web/tests/vm-provider-errors.test.ts b/web/tests/vm-provider-errors.test.ts index fb797f6d63f7..66afac075b8f 100644 --- a/web/tests/vm-provider-errors.test.ts +++ b/web/tests/vm-provider-errors.test.ts @@ -1,10 +1,34 @@ import { describe, expect, test } from "bun:test"; +import { FreestyleApiError } from "freestyle"; +import { ProviderError } from "../services/vms/drivers/types"; import { isProviderIdentityNotFoundError, isProviderNotFoundError, } from "../services/vms/providerErrors"; describe("provider error classification", () => { + test.each([401, 403, 429, 500, 502, 503, 504])("HTTP %s takes precedence over wrapped missing-resource diagnostics", (status) => { + const error = new ProviderError("freestyle", "VM not found while reading stats", new FreestyleApiError( + status, { code: "INTERNAL", message: "VM not found in upstream cache" }, + )); + expect(isProviderNotFoundError(error)).toBe(false); + }); + + test("nested structured status takes precedence over a conflicting not-found code", () => { + const error = { code: "NOT_FOUND", cause: { response: { status: 502 } } }; + expect(isProviderNotFoundError(error)).toBe(false); + expect(isProviderNotFoundError({ code: "NOT_FOUND" })).toBe(true); + expect(isProviderNotFoundError({ cause: { statusCode: 404 } })).toBe(true); + }); + + test.each([ + { status: 0, response: { status: 502 } }, + { status: 600, response: { status: 503 } }, + { status: -1, statusCode: 502 }, + ])("ignores invalid status sentinels before selecting a retryable HTTP status: %j", (error) => { + expect(isProviderNotFoundError({ ...error, message: "VM not found while reading stats" })).toBe(false); + }); + test("keeps identity deletion errors out of VM not-found classification", () => { expect(isProviderNotFoundError(new Error("identity does not exist"))).toBe(false); expect(isProviderIdentityNotFoundError(new Error("identity does not exist"))).toBe(true); diff --git a/web/tests/vm-stats-not-found.test.ts b/web/tests/vm-stats-not-found.test.ts new file mode 100644 index 000000000000..2571f9ee1078 --- /dev/null +++ b/web/tests/vm-stats-not-found.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import { Effect, Layer } from "effect"; +import { Freestyle } from "freestyle"; +import { VmBillingGateway, noOpVmBillingGateway } from "../services/vms/billingGateway"; +import { getProvider } from "../services/vms/drivers"; +import { FreestyleProvider } from "../services/vms/drivers/freestyle"; +import { VmDatabaseError } from "../services/vms/errors"; +import { isOperatorFaultVmError } from "../services/vms/observability"; +import { VmProviderGatewayLive } from "../services/vms/providerGateway"; +import { VmRepository, type CloudVmRow, type VmRepositoryShape } from "../services/vms/repository"; +import { vmWorkflowErrorResponse } from "../services/vms/routeHelpers"; +import { getVmStats } from "../services/vms/workflows"; + +// Real SDK response decoding, driver wrapping, Effect gateway and workflow. +// Only the provider HTTP transport and owned-row repository are fixtures. +async function withStatsFixture( + reply: () => Response, + run: (fixture: { + poll: (userId?: string, teamIds?: readonly string[]) => Promise<{ + tag: string; + response: Response; + payload: { error: string; action: string; retryable?: boolean; ui: { title: string; retryable: boolean } }; + }>; + requests: string[]; + writes: unknown[]; + row: CloudVmRow; + }) => Promise, + options: { failObservation?: boolean; team?: boolean } = {}, +) { + const requests: string[] = []; + const writes: unknown[] = []; + const row = { + id: "fixture-row", userId: "fixture-owner", ownerTeamId: options.team ? "fixture-team" : "fixture-owner", + billingTeamId: options.team ? "fixture-team" : null, + provider: "freestyle", providerVmId: "vm-fixture", providerMetadata: {}, status: "running", + } as CloudVmRow; + const client = new Freestyle({ + apiKey: "test-only", + fetch: (async (input, init) => { + const path = new URL(String(input)).pathname; + requests.push(`${init?.method} ${path}`); + expect(`${init?.method} ${path}`).toBe("GET /v5/vms/vm-fixture"); + return reply(); + }) as typeof fetch, + }); + const driver = new FreestyleProvider({ + client: () => client, + resolveDaemonSource: async () => { throw new Error("Stats must not install anything"); }, + }); + const getStats = spyOn(getProvider("freestyle"), "getStats").mockImplementation((id) => driver.getStats(id)); + const repo = new Proxy({ + findUserVm: (input: { userId: string; providerVmId: string }) => Effect.sync(() => + input.userId === row.userId && input.providerVmId === row.providerVmId && row.status !== "destroyed" ? row : null), + markProviderObservedStatus: (input: { id: string; providerVmId: string; status: "destroyed" }) => Effect.suspend(() => { + writes.push(input); + if (options.failObservation) return Effect.fail(new VmDatabaseError({ operation: "fixture", cause: new Error("offline") })); + expect(input).toEqual({ id: row.id, providerVmId: row.providerVmId, status: "destroyed" }); + row.status = input.status; + return Effect.succeed(true); + }), + }, { + get(target, key) { + if (key in target) return Reflect.get(target, key); + throw new Error(`Unexpected repository operation: ${String(key)}`); + }, + }) as unknown as VmRepositoryShape; + const layer = Layer.mergeAll( + Layer.succeed(VmRepository, repo), VmProviderGatewayLive, + Layer.succeed(VmBillingGateway, noOpVmBillingGateway()), + ); + try { + await run({ + row, requests, writes, + poll: async (userId = row.userId, teamIds = options.team ? ["fixture-team"] : []) => { + const result = await Effect.runPromise(Effect.either(getVmStats({ + userId, teamIds, providerVmId: row.providerVmId!, + }).pipe(Effect.provide(layer)))); + expect(result._tag).toBe("Left"); + if (result._tag !== "Left") throw new Error("Expected a fixture failure"); + const response = await vmWorkflowErrorResponse(result.left); + if (!response) throw new Error("Expected the shared HTTP error contract"); + return { tag: result.left._tag, response, payload: await response.json() }; + }, + }); + } finally { + getStats.mockRestore(); + } +} + +const missing = () => Response.json({ code: "NOT_FOUND", message: "not found: vm vm-fixture" }, { status: 404 }); + +describe("stats provider missing classification", () => { + test("typed NOT_FOUND becomes a terminal 404 across repeated polling, retaining the row", async () => { + await withStatsFixture(missing, async ({ poll, row, requests, writes }) => { + for (let attempt = 0; attempt < 3; attempt += 1) { + const { tag, response, payload } = await poll(); + expect(tag).toBe("VmNotFoundError"); + expect(response.status).toBe(404); + expect(response.headers.get("retry-after")).toBeNull(); + expect(payload.error).toBe("vm_not_found"); + expect(payload.ui).toMatchObject({ title: "Cloud VM not found", retryable: false }); + expect(payload.action).not.toMatch(/temporarily|retry|try again/i); + expect(isOperatorFaultVmError({ error: payload.error, status: response.status })).toBe(false); + } + expect(requests).toHaveLength(1); + expect(writes).toHaveLength(1); + expect(row).toMatchObject({ id: "fixture-row", providerVmId: "vm-fixture", status: "destroyed" }); + }); + }); + + test("a failed observation write never changes missing VM guidance to a retryable outage", async () => { + await withStatsFixture(missing, async ({ poll, row, requests }) => { + for (let attempt = 0; attempt < 3; attempt += 1) { + const { response, payload } = await poll(); + expect(response.status).toBe(404); + expect(payload.ui.retryable).toBe(false); + } + expect(requests).toHaveLength(3); + expect(row.status).toBe("running"); + }, { failObservation: true }); + }); + + test.each([false, true])("ownership is checked before the provider (team=%s)", async (team) => { + await withStatsFixture(missing, async ({ poll, requests, writes }) => { + const { response, payload } = await (team ? poll("fixture-owner", []) : poll("another-user")); + expect(response.status).toBe(404); + expect(payload.error).toBe("vm_not_found"); + expect(JSON.stringify(payload)).not.toMatch(/fixture-owner|fixture-team|fixture-row|freestyle/); + expect(requests).toHaveLength(0); + expect(writes).toHaveLength(0); + }, { team }); + }); + + test.each([ + { name: "typed upstream 502", reply: () => Response.json({ code: "INTERNAL", message: "upstream unavailable" }, { status: 502 }) }, + { name: "typed upstream 502 mentioning a missing VM", reply: () => Response.json({ code: "INTERNAL", message: "VM not found in upstream cache; retry" }, { status: 502 }) }, + { name: "transport failure", reply: () => { throw new TypeError("fetch failed"); } }, + ])("$name remains retryable and cannot mark the machine destroyed", async ({ reply }) => { + await withStatsFixture(reply, async ({ poll, row, writes }) => { + const { tag, response, payload } = await poll(); + expect(tag).toBe("VmProviderOperationError"); + expect(response.status).toBe(502); + expect(payload).toMatchObject({ error: "vm_cloud_service_unavailable", retryable: true, ui: { retryable: true } }); + expect(isOperatorFaultVmError({ error: payload.error, status: response.status })).toBe(true); + expect(writes).toHaveLength(0); + expect(row.status).toBe("running"); + }); + }); +});