diff --git a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx index a3ab8a682ebac..dc348c740f9c4 100644 --- a/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx +++ b/apps/mobile/src/features/connection/CloudEnvironmentRows.tsx @@ -28,6 +28,7 @@ import { environmentPresentations } from "../../state/presentation"; import { availableCloudEnvironmentPresentation } from "../cloud/cloudEnvironmentPresentation"; import { hasCloudPublicConfig } from "../cloud/publicConfig"; import { ConnectionStatusDot } from "./ConnectionStatusDot"; +import { useConnectionDiagnosis } from "./ConnectionDiagnosisDetails"; import { environmentStatusText } from "./environmentStatusText"; import { type RelayEnvironmentView, useConnectionController } from "./useConnectionController"; @@ -193,6 +194,7 @@ function ConnectedCloudEnvironmentRow(props: { ); const canRetry = presentation !== null && hasSavedBearerRoute(presentation.entry); const enabled = props.environment.isEnabled && !unsupported; + const { diagnosis } = useConnectionDiagnosis(props.environment.environmentId, presentation); const [lastDescriptor, setLastDescriptor] = useState(props.descriptor); if (props.descriptor !== undefined && props.descriptor !== lastDescriptor) { setLastDescriptor(props.descriptor); @@ -213,7 +215,11 @@ function ConnectedCloudEnvironmentRow(props: { onValueChange={props.onSetEnabled} onToggleError={props.onToggleError} disabled={unsupported && !canRetry} - {...(enabled || unsupported ? {} : { statusText: "Off" })} + {...(enabled || unsupported + ? diagnosis + ? { statusText: diagnosis.headline } + : {} + : { statusText: "Off" })} value={enabled} /> ); diff --git a/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx b/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx new file mode 100644 index 0000000000000..0fbfd68b480e5 --- /dev/null +++ b/apps/mobile/src/features/connection/ConnectionDiagnosisDetails.tsx @@ -0,0 +1,92 @@ +import { useAtomValue } from "@effect/atom-react"; +import { + AVAILABLE_CONNECTION_STATE, + type ConnectionDiagnosis, + connectionDiagnosisText, + connectionRoutes, + hasRelayRoute, + presentConnectionDiagnosis, + type EnvironmentPresentation, +} from "@t3tools/client-runtime/connection"; +import type { EnvironmentId } from "@t3tools/contracts"; +import Constants from "expo-constants"; +import * as Option from "effect/Option"; +import { AsyncResult } from "effect/reactivity"; +import { View } from "react-native"; + +import { AppText as Text } from "../../components/ui/AppText"; +import { Button } from "../../components/ui/Button"; +import { environmentCatalog } from "../../connection/catalog"; +import { cn } from "../../lib/cn"; +import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; + +const HOP_MARK = { ok: "✓", failed: "✕", unchecked: "–" } as const; +const HOP_NOTE = { ok: "", failed: " · Failed", unchecked: " · Not checked" } as const; + +/** Same model as the web Settings row; null while connected, off, or not failed. */ +export function useConnectionDiagnosis( + environmentId: EnvironmentId, + presentation: EnvironmentPresentation | null, +) { + const state = Option.getOrElse( + AsyncResult.value(useAtomValue(environmentCatalog.stateAtom(environmentId))), + () => AVAILABLE_CONNECTION_STATE, + ); + if (presentation === null || !presentation.entry.enabled) return { diagnosis: null, state }; + const phase = presentation.connection.phase; + if (phase === "connected" || phase === "unsupported") return { diagnosis: null, state }; + const relayOnly = + connectionRoutes(presentation.entry).length === 1 && hasRelayRoute(presentation.entry); + const diagnosis = presentConnectionDiagnosis({ + hostLabel: presentation.entry.target.label, + state, + via: relayOnly ? "relay" : "direct", + clientVersion: Constants.expoConfig?.version, + serverVersion: presentation.serverConfig?.environment.serverVersion, + }); + return { diagnosis, state }; +} + +export function ConnectionDiagnosisDetails(props: { + readonly hostLabel: string; + readonly diagnosis: ConnectionDiagnosis; +}) { + const { diagnosis } = props; + return ( + + + {diagnosis.hops.map((hop) => ( + + {`${HOP_MARK[hop.result]} ${hop.label}${HOP_NOTE[hop.result]}`} + + ))} + + + What to try + {diagnosis.steps.map((step, index) => ( + + {`${index + 1}. ${step}`} + + ))} + + + {diagnosis.facts.map(([label, value]) => `${label}: ${value}`).join("\n")} + + +

+ + ); +} + +/** Hops in order, what to try, the facts, and the actions. */ +export function ConnectionDiagnosisPanel({ + hostLabel, + diagnosis, + onRetry, + onAddRoute, +}: { + readonly hostLabel: string; + readonly diagnosis: ConnectionDiagnosis; + readonly onRetry: () => void; + readonly onAddRoute: (() => void) | null; +}) { + const { copyToClipboard } = useCopyToClipboard({ + target: "connection diagnostics", + onCopy: () => toastManager.add({ type: "success", title: "Diagnostics copied" }), + onError: (error) => + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Could not copy diagnostics", + description: error.message, + }), + ), + }); + return ( +
+
    + {diagnosis.hops.map((hop) => ( +
  1. + + {HOP_MARK[hop.result]} + + {hop.label} + {HOP_NOTE[hop.result]} +
    2. + ))} +
+
+

What to try

+
    + {diagnosis.steps.map((step) => ( +
  1. {step}
    2. + ))} +
+
+
+ {diagnosis.facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+
+ + + {onAddRoute ? ( + + ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/settings/ConnectionsSettings.tsx b/apps/web/src/components/settings/ConnectionsSettings.tsx index a3759c0dbeb48..5585b138dc349 100644 --- a/apps/web/src/components/settings/ConnectionsSettings.tsx +++ b/apps/web/src/components/settings/ConnectionsSettings.tsx @@ -85,6 +85,11 @@ import { LocalEnvironmentSetting } from "./LocalEnvironmentSetting"; import { searchableSetting } from "./settingsSearch"; import { EnvironmentIconMenu } from "./EnvironmentIconPicker"; import { EnvironmentRoutesList } from "./EnvironmentRoutesList"; +import { + ConnectionDiagnosisPanel, + ConnectionDiagnosisSummary, + useConnectionDiagnosis, +} from "./ConnectionDiagnosisDetails"; import { usePreparedConnection } from "~/state/session"; import { EnvironmentRow, @@ -1468,6 +1473,7 @@ function SavedBackendListRow({ }: SavedBackendListRowProps) { const [routesOpen, setRoutesOpen] = useState(false); const [permissionsOpen, setPermissionsOpen] = useState(false); + const [diagnosisOpen, setDiagnosisOpen] = useState(false); const environmentId = environment.environmentId; const unsupported = environment.connection.phase === "unsupported"; const blocked = unsupported && !hasSavedBearerRoute(environment.entry); @@ -1541,6 +1547,13 @@ function SavedBackendListRow({ ) { setLastRelayHttpBaseUrl(discoveredRelayHttpBaseUrl); } + const retryNow = useAtomCommand(environmentCatalog.retryNow); + const { + diagnosis: currentDiagnosis, + state: connectionState, + relayOnly, + } = useConnectionDiagnosis(environment); + const diagnosis = resumingServerUpdate ? null : currentDiagnosis; const prepared = usePreparedConnection(environmentId); const connectedTarget = isConnected && prepared._tag === "Some" ? prepared.value.target : null; const mcpUrl = environmentMcpUrl({ @@ -1555,7 +1568,8 @@ function SavedBackendListRow({ const routeCount = connectionRoutes(environment.entry).length; const subtitleText = [ environmentTransportLabel(environment, connectedTarget), - resumingServerUpdate ? "Restarting" : status.text, + // The headline below names the failure in full. + diagnosis ? null : resumingServerUpdate ? "Restarting" : status.text, enabled && versionMismatch ? serverVersion : null, ] .filter((value): value is string => value !== null) @@ -1598,6 +1612,7 @@ function SavedBackendListRow({ enabled && status.tone === "error" && !resumingServerUpdate && + !diagnosis && "text-destructive", )} /> @@ -1649,14 +1664,33 @@ function SavedBackendListRow({ } below={ - serverUpdateState.status !== "idle" ? ( -
- -
- ) : null + <> + {diagnosis ? ( + setDiagnosisOpen((open) => !open)} + /> + ) : null} + {serverUpdateState.status !== "idle" ? ( +
+ +
+ ) : null} + } detail={ <> + {diagnosis && diagnosisOpen ? ( + void retryNow(environmentId)} + onAddRoute={relayOnly ? () => onAddRoute(environment) : null} + /> + ) : null} {routesOpen && ( { }).pipe(Effect.provide(layerConnectorTest(execute))); }); + it.effect("names a dead tunnel when Cloudflare has no connector for it", () => { + const execute = (request: HttpClientRequest.HttpClientRequest) => + Effect.succeed( + HttpClientResponse.fromWeb(request, new Response("error code: 1033", { status: 530 })), + ); + + return Effect.gen(function* () { + const connector = yield* EnvironmentConnector.EnvironmentConnector; + const connect = yield* connector + .connect({ + userId: "user_123", + environmentId: "env-connector-test", + clientProofKeyThumbprint: "client-proof-key-thumbprint", + }) + .pipe(Effect.flip); + const status = yield* connector.status({ + userId: "user_123", + environmentId: "env-connector-test", + }); + + expect(connect).toMatchObject({ + _tag: "EnvironmentMintRequestFailed", + endpointFailure: "tunnel_down", + endpointStatus: 530, + }); + expect(status).toMatchObject({ + status: "offline", + endpointFailure: "tunnel_down", + endpointStatus: 530, + }); + }).pipe(Effect.provide(layerConnectorTest(execute))); + }); + it.effect("times out hung managed endpoint mint requests", () => { let resolveRequestStarted: (() => void) | undefined; const requestStarted = new Promise((resolve) => { diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts index 1156206f6e182..a6a64ba3dfda5 100644 --- a/infra/relay/src/environments/EnvironmentConnector.ts +++ b/infra/relay/src/environments/EnvironmentConnector.ts @@ -14,6 +14,7 @@ import { RelayEnvironmentMintResponseProofPayload, RelayCloudMintCredentialProofPayload, RelayEnvironmentConnectNotAuthorizedReason, + RelayEnvironmentEndpointFailure, type RelayEnvironmentConnectResponse, type RelayEnvironmentStatusResponse, type RelayManagedEndpoint, @@ -42,6 +43,7 @@ import * as FetchHttpClient from "effect/http/FetchHttpClient"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientError from "effect/http/HttpClientError"; +import { diagnoseEndpointFailure } from "./endpointFailure.ts"; import * as EnvironmentLinks from "./EnvironmentLinks.ts"; import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as RelayConfiguration from "../Config.ts"; @@ -89,6 +91,8 @@ export class EnvironmentMintRequestFailed extends Schema.TaggedError - new EnvironmentMintRequestFailed({ - environmentId: input.environmentId, - operation: "connect", - cause, - }), + Effect.catch((cause) => + diagnoseEndpointFailure(cause).pipe( + Effect.flatMap((diagnosis) => + Effect.fail( + new EnvironmentMintRequestFailed({ + environmentId: input.environmentId, + operation: "connect", + cause, + ...diagnosis, + }), + ), + ), + ), ), Effect.timeoutOption(Duration.millis(ENVIRONMENT_MINT_REQUEST_TIMEOUT_MS)), Effect.flatMap( diff --git a/infra/relay/src/environments/endpointFailure.test.ts b/infra/relay/src/environments/endpointFailure.test.ts new file mode 100644 index 0000000000000..bce7617252776 --- /dev/null +++ b/infra/relay/src/environments/endpointFailure.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { classifyEndpointResponse, classifyTransportFailure } from "./endpointFailure.ts"; + +describe("classifyEndpointResponse", () => { + it.each([ + [530, "error code: 1033", "tunnel_down"], + [530, "", "tunnel_down"], + [530, "error code: 1016", "dns"], + [502, "Bad Gateway", "origin_unreachable"], + [503, "", "origin_unreachable"], + [521, "error code: 521", "refused"], + [524, "", "timeout"], + [526, "", "tls"], + [404, "Not Found", "invalid_response"], + [200, "", "invalid_response"], + ] as const)("maps HTTP %i with body %j to %s", (status, body, expected) => { + expect(classifyEndpointResponse(status, body)).toBe(expected); + }); +}); + +describe("classifyTransportFailure", () => { + const nested = (code: string) => + new TypeError("fetch failed", { cause: Object.assign(new Error(code), { code }) }); + + it.each([ + ["ENOTFOUND", "dns"], + ["ECONNREFUSED", "refused"], + ["UNABLE_TO_VERIFY_LEAF_SIGNATURE: certificate", "tls"], + ["UND_ERR_CONNECT_TIMEOUT", "timeout"], + ["ECONNRESET", "unknown"], + ] as const)("reads %s from the cause chain", (code, expected) => { + expect(classifyTransportFailure(nested(code))).toBe(expected); + }); +}); diff --git a/infra/relay/src/environments/endpointFailure.ts b/infra/relay/src/environments/endpointFailure.ts new file mode 100644 index 0000000000000..2d4983c29405b --- /dev/null +++ b/infra/relay/src/environments/endpointFailure.ts @@ -0,0 +1,93 @@ +import type { RelayEnvironmentEndpointFailure } from "@t3tools/contracts/relay"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as HttpClientError from "effect/http/HttpClientError"; + +export interface EndpointFailureDiagnosis { + readonly endpointFailure: RelayEnvironmentEndpointFailure; + readonly endpointStatus?: number; +} + +/** + * Cloudflare answers a Worker subrequest it cannot route with a plain-text body + * such as `error code: 1033`. 1033 means no cloudflared connector is running for + * the tunnel; 1016 means the origin hostname did not resolve. + */ +export function classifyEndpointResponse( + status: number, + body: string, +): RelayEnvironmentEndpointFailure { + const cloudflareCode = /error code:\s*(\d{4})/i.exec(body)?.[1]; + if (cloudflareCode === "1016") return "dns"; + if (cloudflareCode === "1033") return "tunnel_down"; + switch (status) { + case 530: + return cloudflareCode === undefined ? "tunnel_down" : "unknown"; + // cloudflared answers 502 when nothing listens on the local origin port. + case 502: + case 503: + return "origin_unreachable"; + case 521: + return "refused"; + case 504: + case 522: + case 524: + return "timeout"; + case 525: + case 526: + return "tls"; + default: + // Any other status came from T3 Code itself, in a shape we did not expect. + return "invalid_response"; + } +} + +/** Reads the error code from the transport failure's cause chain, such as undici's `ECONNREFUSED`. */ +export function classifyTransportFailure(cause: unknown): RelayEnvironmentEndpointFailure { + const parts: Array = []; + for (let current = cause, depth = 0; current instanceof Error && depth < 4; depth++) { + const code = "code" in current ? current.code : undefined; + parts.push(current.message, typeof code === "string" ? code : ""); + current = current.cause; + } + const text = parts.join(" "); + if (/ENOTFOUND|EAI_AGAIN|getaddrinfo|\bDNS\b/i.test(text)) return "dns"; + if (/ECONNREFUSED|refused/i.test(text)) return "refused"; + if (/CERT|\bTLS\b|\bSSL\b|certificate/i.test(text)) return "tls"; + if (/ETIMEDOUT|TIMEOUT|timed out/i.test(text)) return "timeout"; + return "unknown"; +} + +const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2); + +/** Turns a failed environment request into the cause the client shows the user. */ +export const diagnoseEndpointFailure = Effect.fnUntraced(function* (cause: unknown) { + if (!HttpClientError.isHttpClientError(cause)) { + return { endpointFailure: "invalid_response" } satisfies EndpointFailureDiagnosis; + } + const response = cause.response; + if (response === undefined) { + return { + endpointFailure: + cause.reason._tag === "TransportError" + ? classifyTransportFailure(cause.reason.cause) + : "unknown", + } satisfies EndpointFailureDiagnosis; + } + // Only Cloudflare's 530 needs the body to tell a dead tunnel from a DNS error. + const body = + response.status === 530 + ? Option.getOrElse( + yield* response.text.pipe( + Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT), + Effect.orElseSucceed(() => Option.none()), + ), + () => "", + ) + : ""; + return { + endpointFailure: classifyEndpointResponse(response.status, body), + endpointStatus: response.status, + } satisfies EndpointFailureDiagnosis; +}); diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 0618ee9c65378..6c7ff28cb9235 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -1105,10 +1105,12 @@ export const layerDpopClientApi = HttpApiBuilder.group( reason: error.reason, traceId, }), - EnvironmentMintRequestFailed: (_error, traceId) => + EnvironmentMintRequestFailed: (error, traceId) => new RelayEnvironmentEndpointUnavailableError({ code: "environment_endpoint_unavailable", reason: "endpoint_request_failed", + ...(error.endpointFailure ? { endpointFailure: error.endpointFailure } : {}), + ...(error.endpointStatus ? { endpointStatus: error.endpointStatus } : {}), traceId, }), EnvironmentMintRequestTimedOut: (_error, traceId) => @@ -1148,10 +1150,12 @@ export const layerDpopClientApi = HttpApiBuilder.group( reason: error.reason, traceId, }), - EnvironmentMintRequestFailed: (_error, traceId) => + EnvironmentMintRequestFailed: (error, traceId) => new RelayEnvironmentEndpointUnavailableError({ code: "environment_endpoint_unavailable", reason: "endpoint_request_failed", + ...(error.endpointFailure ? { endpointFailure: error.endpointFailure } : {}), + ...(error.endpointStatus ? { endpointStatus: error.endpointStatus } : {}), traceId, }), EnvironmentMintRequestTimedOut: (_error, traceId) => diff --git a/packages/client-runtime/src/connection/diagnosis.test.ts b/packages/client-runtime/src/connection/diagnosis.test.ts new file mode 100644 index 0000000000000..babfccb636d48 --- /dev/null +++ b/packages/client-runtime/src/connection/diagnosis.test.ts @@ -0,0 +1,207 @@ +import { + RelayAuthInvalidError, + RelayEnvironmentEndpointTimedOutError, + RelayEnvironmentEndpointUnavailableError, + type RelayProtectedError, +} from "@t3tools/contracts/relay"; +import { describe, expect, it } from "@effect/vitest"; + +import { connectionDiagnosisText, presentConnectionDiagnosis } from "./diagnosis.ts"; +import { + credentialMissingError, + mapManagedRelayError, + mapRemoteEnvironmentError, +} from "./errors.ts"; +import { + AVAILABLE_CONNECTION_STATE, + type ConnectionAttemptError, + type SupervisorConnectionState, +} from "./model.ts"; +import { ManagedRelayRequestFailedError } from "../relay/managedRelay.ts"; +import { RemoteEnvironmentAuthFetchError } from "../rpc/http.ts"; + +const backoff = (lastFailure: ConnectionAttemptError): SupervisorConnectionState => ({ + ...AVAILABLE_CONNECTION_STATE, + desired: true, + network: "online", + phase: "backoff", + attempt: 3, + lastFailure, + retryAt: Date.UTC(2026, 9, 19, 12, 0, 30), +}); + +const fromRelay = (relayError: RelayProtectedError) => + mapManagedRelayError( + new ManagedRelayRequestFailedError({ + action: "connect relay environment", + cause: relayError, + relayError, + }), + ); + +const unavailable = (endpointFailure: "tunnel_down" | "origin_unreachable") => + fromRelay( + new RelayEnvironmentEndpointUnavailableError({ + code: "environment_endpoint_unavailable", + reason: "endpoint_request_failed", + endpointFailure, + endpointStatus: endpointFailure === "tunnel_down" ? 530 : 502, + traceId: "trace-1", + }), + ); + +const diagnose = (state: SupervisorConnectionState, via: "relay" | "direct" = "relay") => + presentConnectionDiagnosis({ hostLabel: "Mac mini", state, via }); + +const results = (state: SupervisorConnectionState, via?: "relay" | "direct") => + diagnose(state, via)?.hops.map((hop) => `${hop.hop}:${hop.result}`); + +describe("presentConnectionDiagnosis", () => { + it("names a dead tunnel and points the fix at the host", () => { + const diagnosis = diagnose(backoff(unavailable("tunnel_down"))); + expect(diagnosis?.headline).toBe("Mac mini's tunnel is offline"); + expect(results(backoff(unavailable("tunnel_down")))).toEqual([ + "device:ok", + "relay:ok", + "tunnel:failed", + "server:unchecked", + ]); + expect(diagnosis?.steps.join(" ")).toContain("Tailscale exit node on Mac mini"); + expect(diagnosis?.facts).toContainEqual(["HTTP status", "530"]); + expect(diagnosis?.facts).toContainEqual(["Trace ID", "trace-1"]); + }); + + it("tells a working tunnel apart from T3 Code not answering", () => { + expect(diagnose(backoff(unavailable("origin_unreachable")))?.headline).toBe( + "Tunnel works, but T3 Code on Mac mini did not answer", + ); + expect(results(backoff(unavailable("origin_unreachable")), "direct")).toEqual([ + "device:ok", + "relay:ok", + "tunnel:ok", + "server:failed", + ]); + }); + + it("blames this device when it cannot reach a tunnel the relay already reached", () => { + const failure = mapRemoteEnvironmentError( + new RemoteEnvironmentAuthFetchError({ message: "Failed to fetch", cause: new TypeError() }), + "relay", + ); + // A learned LAN route makes the environment's usual route direct; the failure still names the relay. + const diagnosis = diagnose(backoff(failure), "direct"); + expect(diagnosis?.headline).toBe("This device can't reach Mac mini's tunnel"); + expect(diagnosis?.hops).toHaveLength(4); + expect(diagnosis?.steps.join(" ")).not.toContain("On Mac mini"); + }); + + it("points a relay timeout at the host, not this device", () => { + const failure = fromRelay( + new RelayEnvironmentEndpointTimedOutError({ + code: "environment_endpoint_timed_out", + traceId: "trace-3", + }), + ); + const diagnosis = diagnose(backoff(failure)); + expect(diagnosis?.headline).toBe("Mac mini did not answer in time"); + expect(diagnosis?.steps.join(" ")).toContain("On Mac mini"); + expect(diagnosis?.steps.join(" ")).not.toContain("on this device"); + }); + + it("stops at T3 Connect when this device cannot reach the relay", () => { + const failure = mapManagedRelayError( + new ManagedRelayRequestFailedError({ + action: "connect relay environment", + transportFailed: true, + cause: new TypeError("Failed to fetch"), + }), + ); + expect(diagnose(backoff(failure))?.headline).toBe("This device can't reach T3 Connect"); + expect(results(backoff(failure))).toEqual([ + "device:ok", + "relay:failed", + "tunnel:unchecked", + "server:unchecked", + ]); + }); + + it("does not blame this device's network when T3 Connect answered with an error", () => { + const failure = mapManagedRelayError( + new ManagedRelayRequestFailedError({ + action: "connect relay environment", + cause: new Error("Bad Gateway"), + status: 502, + }), + ); + const diagnosis = diagnose(backoff(failure)); + expect(diagnosis?.headline).toBe("T3 Connect returned an error"); + expect(diagnosis?.steps.join(" ")).not.toContain("this device"); + }); + + it("blames this device when its saved credential is missing", () => { + const diagnosis = diagnose(backoff(credentialMissingError("connection-1")), "direct"); + expect(diagnosis?.hops[0]).toMatchObject({ hop: "device", result: "failed" }); + }); + + it("names clock skew from a DPoP time window rejection", () => { + const failure = fromRelay( + new RelayAuthInvalidError({ + code: "auth_invalid", + reason: "invalid_dpop", + dpopFailureReason: "time_window", + traceId: "trace-2", + }), + ); + expect(diagnose(backoff(failure))?.headline).toBe("This device's clock is off"); + }); + + it("reports this device as offline before any other hop", () => { + const state = { ...backoff(unavailable("tunnel_down")), network: "offline" as const }; + expect(diagnose(state)?.headline).toBe("This device is offline"); + expect(results(state)?.[0]).toBe("device:failed"); + }); + + it("uses the relay health check before any connect attempt fails", () => { + const diagnosis = presentConnectionDiagnosis({ + hostLabel: "Mac mini", + state: AVAILABLE_CONNECTION_STATE, + via: "relay", + relayStatus: { status: "offline", endpointFailure: "tunnel_down", endpointStatus: 530 }, + }); + expect(diagnosis?.headline).toBe("Mac mini's tunnel is offline"); + const origin = presentConnectionDiagnosis({ + hostLabel: "Mac mini", + state: AVAILABLE_CONNECTION_STATE, + via: "direct", + relayStatus: { status: "offline", endpointFailure: "origin_unreachable" }, + }); + expect(origin?.hops.map((hop) => hop.hop)).toEqual(["device", "relay", "tunnel", "server"]); + }); + + it("keeps the relay health status out of a failure that has its own cause", () => { + const failure = mapRemoteEnvironmentError( + new RemoteEnvironmentAuthFetchError({ message: "Failed to fetch", cause: new TypeError() }), + ); + const diagnosis = presentConnectionDiagnosis({ + hostLabel: "Mac mini", + state: backoff(failure), + via: "direct", + relayStatus: { status: "offline", endpointFailure: "tunnel_down", endpointStatus: 530 }, + }); + expect(diagnosis?.facts.map(([label]) => label)).not.toContain("HTTP status"); + }); + + it("has nothing to say when nothing failed", () => { + expect(diagnose(AVAILABLE_CONNECTION_STATE)).toBeNull(); + }); + + it("writes every hop and fact into the copied diagnostics", () => { + const diagnosis = diagnose(backoff(unavailable("tunnel_down"))); + if (diagnosis === null) throw new Error("expected a diagnosis"); + const text = connectionDiagnosisText("Mac mini", diagnosis); + expect(text).toContain("Mac mini tunnel: FAILED"); + expect(text).toContain("T3 Code on Mac mini: not checked"); + expect(text).toContain("Cause: tunnel_down"); + expect(text).toContain("Attempts: 3"); + }); +}); diff --git a/packages/client-runtime/src/connection/diagnosis.ts b/packages/client-runtime/src/connection/diagnosis.ts new file mode 100644 index 0000000000000..96db26f99be15 --- /dev/null +++ b/packages/client-runtime/src/connection/diagnosis.ts @@ -0,0 +1,288 @@ +import type { RelayEnvironmentStatusResponse } from "@t3tools/contracts/relay"; +import * as DateTime from "effect/DateTime"; + +import type { + ConnectionAttemptError, + ConnectionFailureCause, + ConnectionHop, + SupervisorConnectionState, +} from "./model.ts"; + +export type ConnectionHopResult = "ok" | "failed" | "unchecked"; + +export interface ConnectionDiagnosisHop { + readonly hop: ConnectionHop; + readonly label: string; + readonly result: ConnectionHopResult; +} + +export interface ConnectionDiagnosis { + /** Names the part that failed, e.g. "Mac mini's tunnel is offline". */ + readonly headline: string; + readonly hops: ReadonlyArray; + readonly steps: ReadonlyArray; + readonly facts: ReadonlyArray; +} + +export interface ConnectionDiagnosisInput { + readonly hostLabel: string; + readonly state: SupervisorConnectionState; + /** The environment's usual route, used when the failure does not name one. */ + readonly via: "relay" | "direct"; + /** The relay's last health check, which can name a cause before any connect attempt. */ + readonly relayStatus?: Pick< + RelayEnvironmentStatusResponse, + "status" | "endpointFailure" | "endpointStatus" | "offlineReason" + > | null; + readonly clientVersion?: string | undefined; + readonly serverVersion?: string | null | undefined; +} + +interface Located { + readonly hop: ConnectionHop; + readonly cause: ConnectionFailureCause | "offline" | "outdated" | "released" | null; +} + +const NOT_ANSWERING: ReadonlySet = new Set([ + "origin_unreachable", + "invalid_response", +]); + +function locate(input: ConnectionDiagnosisInput): Located | null { + const failure = input.state.lastFailure; + if (input.state.network === "offline" || input.state.phase === "offline") { + return { hop: "device", cause: "offline" }; + } + if (failure?._tag === "ConnectionBlockedError" && failure.serverUpdateRequired) { + return { hop: "server", cause: "outdated" }; + } + if (failure?.hop !== undefined) { + return { hop: failure.hop, cause: failure.failureCause ?? null }; + } + if (failure !== null) { + const unreachable = + failure.reason === "network" || + failure.reason === "timeout" || + failure.reason === "transport"; + const via = failure.via ?? input.via; + return { hop: unreachable && via === "relay" ? "tunnel" : "server", cause: null }; + } + const status = input.relayStatus; + if (status?.status !== "offline") return null; + if (status.offlineReason === "tunnel_released") return { hop: "tunnel", cause: "released" }; + const cause = status.endpointFailure ?? null; + return { hop: cause !== null && NOT_ANSWERING.has(cause) ? "server" : "tunnel", cause }; +} + +function headline(host: string, located: Located, failure: ConnectionAttemptError | null): string { + switch (located.cause) { + case "offline": + return "This device is offline"; + case "outdated": + return `${host} runs an older T3 Code`; + case "clock_skew": + return "This device's clock is off"; + case "not_linked": + return `${host} is not linked to your T3 Connect account`; + case "relay_error": + return "T3 Connect returned an error"; + case "tunnel_pending": + return `${host}'s tunnel is still being set up`; + case "tunnel_not_enabled": + return `T3 Connect is not turned on for ${host}`; + case "released": + case "tunnel_down": + return `${host}'s tunnel is offline`; + case "origin_unreachable": + return `Tunnel works, but T3 Code on ${host} did not answer`; + case "invalid_response": + return `T3 Code on ${host} sent an unexpected answer`; + case "dns": + return `${host}'s tunnel address did not resolve`; + case "tls": + return `Secure connection to ${host}'s tunnel failed`; + case "refused": + return `${host}'s tunnel refused the connection`; + case "timeout": + return `${host} did not answer in time`; + case "unknown": + case null: + break; + } + switch (located.hop) { + case "device": + return "This device could not prepare the connection"; + case "relay": + return failure?.reason === "authentication" + ? "T3 Connect did not accept this device's sign-in" + : "This device can't reach T3 Connect"; + case "tunnel": + // The relay already reached the host when this device fails on its own (#17629). + return failure?.reason === "network" || + failure?.reason === "transport" || + failure?.reason === "timeout" + ? `This device can't reach ${host}'s tunnel` + : `T3 Connect could not reach ${host}'s tunnel`; + case "server": + return failure?.reason === "authentication" || failure?.reason === "permission" + ? `T3 Code on ${host} did not accept this device` + : `T3 Code on ${host} did not answer`; + } +} + +function steps(host: string, located: Located, failure: ConnectionAttemptError | null): string[] { + const checkHost = `On ${host}, check Settings > Connections or run \`t3 connect status\`.`; + switch (located.cause) { + case "offline": + return ["Connect this device to the internet. T3 Code reconnects on its own."]; + case "outdated": + return [`Update T3 Code on ${host}.`]; + case "clock_skew": + return [`Turn on automatic date and time on this device and on ${host}, then retry.`]; + case "not_linked": + return [ + "Check that this device is signed in to the same T3 Connect account as the host.", + `On ${host}, open Settings > Connections and turn T3 Connect on.`, + ]; + case "tunnel_pending": + return ["Wait a minute. T3 Code retries on its own.", checkHost]; + case "relay_error": + return [ + "Wait a few minutes. T3 Code retries on its own.", + "If it keeps failing, copy the diagnostics and report it.", + ]; + case "tunnel_not_enabled": + return [`On ${host}, open Settings > Connections and turn T3 Connect on.`]; + case "released": + case "tunnel_down": + return [ + `Make sure ${host} is awake and T3 Code is running on it.`, + checkHost, + `A VPN or Tailscale exit node on ${host} can block the tunnel (port 7844). Turn it off or allow the tunnel.`, + ]; + case "origin_unreachable": + case "invalid_response": + return [`Open or restart T3 Code on ${host}.`, checkHost]; + // The relay saw these on its way to the host, so the fix is on the host. + case "dns": + case "tls": + case "refused": + case "timeout": + case "unknown": + return [`Make sure ${host} is awake and online.`, checkHost]; + case null: + break; + } + switch (located.hop) { + case "device": + return [ + "Restart T3 Code on this device.", + `If it keeps failing, remove ${host} from this device and add it again.`, + ]; + case "relay": + return failure?.reason === "authentication" + ? ["Sign out of T3 Connect and sign in again."] + : [ + "Check this device's internet connection.", + "A VPN, firewall, or ad blocker on this device can block T3 Connect.", + ]; + case "tunnel": + return failure?.reason === "network" || + failure?.reason === "transport" || + failure?.reason === "timeout" + ? [ + "A VPN, firewall, or DNS filter on this device can block the tunnel.", + "Try another network, or add a direct route such as a Tailscale address.", + ] + : [`Make sure ${host} is awake and online.`, checkHost]; + case "server": + return [ + `Make sure T3 Code is running on ${host}.`, + "Retry. If it keeps failing, copy the diagnostics.", + ]; + } +} + +const HOP_ORDER = { + relay: ["device", "relay", "tunnel", "server"], + direct: ["device", "server"], +} as const; + +function hopLabel(hop: ConnectionHop, host: string): string { + switch (hop) { + case "device": + return "This device"; + case "relay": + return "T3 Connect"; + case "tunnel": + return `${host} tunnel`; + case "server": + return `T3 Code on ${host}`; + } +} + +/** + * Turns a connection state into what failed, in order, and what to try. Returns + * null when nothing has failed. Web and mobile render this same model. + */ +export function presentConnectionDiagnosis( + input: ConnectionDiagnosisInput, +): ConnectionDiagnosis | null { + const located = locate(input); + if (located === null) return null; + const host = input.hostLabel; + const failure = input.state.lastFailure; + const relayPath = + located.hop === "relay" || + located.hop === "tunnel" || + (failure === null && input.relayStatus?.status === "offline") || + (failure?.via ?? input.via) === "relay"; + const order: ReadonlyArray = HOP_ORDER[relayPath ? "relay" : "direct"]; + const failedIndex = order.indexOf(located.hop); + const hops = order.map((hop, index): ConnectionDiagnosisHop => ({ + hop, + label: hopLabel(hop, host), + result: index < failedIndex ? "ok" : index === failedIndex ? "failed" : "unchecked", + })); + + // The relay health check only explains the state when no attempt has failed yet. + const httpStatus = failure !== null ? failure.httpStatus : input.relayStatus?.endpointStatus; + const facts: Array = []; + const add = (label: string, value: string | number | null | undefined) => { + if (value !== null && value !== undefined && value !== "") facts.push([label, String(value)]); + }; + add("Cause", located.cause ?? failure?.reason); + add("HTTP status", httpStatus); + add("Attempts", input.state.attempt > 0 ? input.state.attempt : null); + add( + "Next retry", + input.state.retryAt === null + ? null + : DateTime.formatIso(DateTime.makeUnsafe(input.state.retryAt)), + ); + add("Route", relayPath ? "T3 Connect" : "Direct"); + add("App version", input.clientVersion); + add("Host version", input.serverVersion); + add("Error", failure?.detail); + add("Trace ID", failure?.traceId); + + return { + headline: headline(host, located, failure), + hops, + steps: steps(host, located, failure), + facts, + }; +} + +const HOP_MARK = { ok: "ok", failed: "FAILED", unchecked: "not checked" } as const; + +/** One plain-text block for bug reports. */ +export function connectionDiagnosisText(hostLabel: string, diagnosis: ConnectionDiagnosis): string { + return [ + `T3 Code connection to ${hostLabel}: ${diagnosis.headline}`, + "", + ...diagnosis.hops.map((hop) => `${hop.label}: ${HOP_MARK[hop.result]}`), + "", + ...diagnosis.facts.map(([label, value]) => `${label}: ${value}`), + ].join("\n"); +} diff --git a/packages/client-runtime/src/connection/errors.ts b/packages/client-runtime/src/connection/errors.ts index 81ed19419b978..a5961df442f2c 100644 --- a/packages/client-runtime/src/connection/errors.ts +++ b/packages/client-runtime/src/connection/errors.ts @@ -14,6 +14,7 @@ export function profileMissingError(connectionId: string): ConnectionBlockedErro return new ConnectionBlockedError({ reason: "configuration", detail: `Connection profile ${connectionId} is unavailable.`, + hop: "device", }); } @@ -21,6 +22,7 @@ export function credentialMissingError(connectionId: string): ConnectionBlockedE return new ConnectionBlockedError({ reason: "authentication", detail: `Connection credential ${connectionId} is unavailable.`, + hop: "device", }); } @@ -35,15 +37,25 @@ export function environmentMismatchError(input: { } function relayProtectedError(error: RelayProtectedError): ConnectionAttemptError { + const detail = relayProtectedErrorMessage(error); + const traceId = error.traceId; switch (error._tag) { case "RelayAuthInvalidError": + return new ConnectionBlockedError({ + reason: "authentication", + detail, + traceId, + hop: "relay", + ...(error.dpopFailureReason === "time_window" ? { failureCause: "clock_skew" } : {}), + }); case "RelayEnvironmentLinkProofExpiredError": case "RelayAgentActivityPublishProofExpiredError": case "RelayAgentActivityPublishProofInvalidError": return new ConnectionBlockedError({ reason: "authentication", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "relay", }); case "RelayEnvironmentConnectNotAuthorizedError": // A host swapping in a new tunnel leaves its allocation unready until @@ -51,41 +63,72 @@ function relayProtectedError(error: RelayProtectedError): ConnectionAttemptError if (error.reason === "managed_endpoint_allocation_not_ready") { return new ConnectionTransientError({ reason: "endpoint-unavailable", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "tunnel", + failureCause: "tunnel_pending", + }); + } + if (error.reason === "endpoint_provider_not_managed") { + return new ConnectionBlockedError({ + reason: "permission", + detail, + traceId, + hop: "tunnel", + failureCause: "tunnel_not_enabled", }); } return new ConnectionBlockedError({ reason: "permission", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "relay", + ...(error.reason === "environment_link_not_found" ? { failureCause: "not_linked" } : {}), }); case "RelayEnvironmentLinkProofInvalidError": case "RelayEnvironmentLinkLimitExceededError": - return new ConnectionBlockedError({ - reason: "permission", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, - }); + return new ConnectionBlockedError({ reason: "permission", detail, traceId, hop: "relay" }); case "RelayEnvironmentEndpointTimedOutError": return new ConnectionTransientError({ reason: "timeout", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "tunnel", + failureCause: "timeout", + }); + case "RelayEnvironmentEndpointUnavailableError": { + const endpointFailure = + error.endpointFailure ?? + (error.reason === "endpoint_response_invalid" ? "invalid_response" : undefined); + return new ConnectionTransientError({ + reason: "endpoint-unavailable", + detail, + traceId, + // The tunnel carried the request when T3 Code is what failed to answer. + hop: + endpointFailure === "origin_unreachable" || endpointFailure === "invalid_response" + ? "server" + : "tunnel", + via: "relay", + ...(endpointFailure ? { failureCause: endpointFailure } : {}), + ...(error.endpointStatus ? { httpStatus: error.endpointStatus } : {}), }); - case "RelayEnvironmentEndpointUnavailableError": + } case "RelayEnvironmentLinkUnavailableError": return new ConnectionTransientError({ reason: "endpoint-unavailable", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "tunnel", }); case "RelayEnvironmentLinkFailedError": case "RelayInternalError": return new ConnectionTransientError({ reason: "relay-unavailable", - detail: relayProtectedErrorMessage(error), - traceId: error.traceId, + detail, + traceId, + hop: "relay", + failureCause: "relay_error", }); } } @@ -99,22 +142,30 @@ export function mapManagedRelayError(error: ManagedRelayClientError): Connection return new ConnectionTransientError({ reason: "relay-unavailable", detail: error.message, + hop: "relay", + // A status means the relay answered, so this device did reach it. + ...(error.status !== undefined + ? { httpStatus: error.status, failureCause: "relay_error" } + : {}), ...(error.traceId ? { traceId: error.traceId } : {}), }); case "ManagedRelayRequestTimeoutError": return new ConnectionTransientError({ reason: "timeout", detail: error.message, + hop: "relay", }); case "ManagedRelayUrlInvalidError": return new ConnectionBlockedError({ reason: "configuration", detail: error.message, + hop: "relay", }); case "ManagedRelayAccessTokenScopesUnexpectedError": return new ConnectionBlockedError({ reason: "permission", detail: error.message, + hop: "relay", }); case "ManagedRelayDpopKeyLoadError": case "ManagedRelayTokenProofCreationError": @@ -122,6 +173,7 @@ export function mapManagedRelayError(error: ManagedRelayClientError): Connection return new ConnectionBlockedError({ reason: "authentication", detail: error.message, + hop: "device", }); } } @@ -131,12 +183,18 @@ export function mapRemoteEnvironmentError( connectionMethod: ClientConnectionMethod = "direct", ): ConnectionAttemptError { const networkHint = connectionMethod === "relay" ? ` ${NETWORK_BLOCKING_HINT}` : ""; + // Over T3 Connect the relay already reached the host, so a request that never + // arrives was blocked between this device and the tunnel. + const unreachableHop = connectionMethod === "relay" ? "tunnel" : "server"; + const via = connectionMethod === "relay" ? "relay" : "direct"; switch (error._tag) { case "EnvironmentAuthInvalidError": return new ConnectionBlockedError({ reason: "authentication", detail: "The environment credential is invalid.", traceId: error.traceId, + hop: "server", + via, }); case "EnvironmentScopeRequiredError": case "EnvironmentOperationForbiddenError": @@ -144,40 +202,54 @@ export function mapRemoteEnvironmentError( reason: "permission", detail: "The environment credential does not grant the required access.", traceId: error.traceId, + hop: "server", + via, }); case "EnvironmentRequestInvalidError": return new ConnectionBlockedError({ reason: "configuration", detail: "The environment rejected the authentication request.", traceId: error.traceId, + hop: "server", + via, }); case "EnvironmentResourceNotFoundError": return new ConnectionBlockedError({ reason: "configuration", detail: "The environment endpoint could not be found.", traceId: error.traceId, + hop: "server", + via, }); case "RemoteEnvironmentAuthTimeoutError": return new ConnectionTransientError({ reason: "timeout", detail: `${error.message}${networkHint}`, + hop: unreachableHop, + via, }); case "RemoteEnvironmentAuthFetchError": return new ConnectionTransientError({ reason: "network", detail: `${error.message}${networkHint}`, + hop: unreachableHop, + via, }); case "EnvironmentInternalError": return new ConnectionTransientError({ reason: "remote-unavailable", detail: "The environment could not authorize the connection.", traceId: error.traceId, + hop: "server", + via, }); case "RemoteEnvironmentAuthInvalidJsonError": case "RemoteEnvironmentAuthUndeclaredStatusError": return new ConnectionTransientError({ reason: "remote-unavailable", detail: error.message, + hop: "server", + via, }); } } @@ -195,6 +267,9 @@ export function mapRemoteDpopEnvironmentError( reason: "authentication", detail: dpopFailureMessage("The environment credential is invalid.", error.dpopFailureReason), traceId: error.traceId, + hop: "server", + via: "relay", + ...(error.dpopFailureReason === "time_window" ? { failureCause: "clock_skew" } : {}), }); } return mapRemoteEnvironmentError(error, "relay"); diff --git a/packages/client-runtime/src/connection/index.ts b/packages/client-runtime/src/connection/index.ts index 90881817c6420..756e68c5dd450 100644 --- a/packages/client-runtime/src/connection/index.ts +++ b/packages/client-runtime/src/connection/index.ts @@ -1,4 +1,5 @@ export * from "./catalog.ts"; +export * from "./diagnosis.ts"; export * as Connectivity from "./connectivity.ts"; export * as CredentialStore from "./credentialStore.ts"; export { type ConnectionDriverProgress, type EnvironmentConnectionLease } from "./driver.ts"; diff --git a/packages/client-runtime/src/connection/model.ts b/packages/client-runtime/src/connection/model.ts index 7a7ecaf492607..ca3062bf24328 100644 --- a/packages/client-runtime/src/connection/model.ts +++ b/packages/client-runtime/src/connection/model.ts @@ -1,4 +1,5 @@ import { EnvironmentId } from "@t3tools/contracts"; +import { RelayEnvironmentEndpointFailure } from "@t3tools/contracts/relay"; import * as Schema from "effect/Schema"; const ConnectionTargetBase = { @@ -76,12 +77,40 @@ export const ConnectionBlockedReason = Schema.Literals([ ]); export type ConnectionBlockedReason = typeof ConnectionBlockedReason.Type; +/** A part of the path to an environment: this device, T3 Connect, the host's tunnel, or T3 Code on the host. */ +export const ConnectionHop = Schema.Literals(["device", "relay", "tunnel", "server"]); +export type ConnectionHop = typeof ConnectionHop.Type; + +/** The relay's endpoint failures plus the causes the client can name from other errors. */ +export const ConnectionFailureCause = Schema.Literals([ + ...RelayEnvironmentEndpointFailure.literals, + "clock_skew", + "not_linked", + "relay_error", + "tunnel_pending", + "tunnel_not_enabled", +]); +export type ConnectionFailureCause = typeof ConnectionFailureCause.Type; + +/** + * Where and why a failure happened, when the code that saw it knows. See + * `presentConnectionDiagnosis`. `via` names the route when the hop alone does + * not: T3 Code on the host can fail behind the tunnel or on a direct route. + */ +const ConnectionFailureFields = { + hop: Schema.optionalKey(ConnectionHop), + via: Schema.optionalKey(Schema.Literals(["relay", "direct"])), + failureCause: Schema.optionalKey(ConnectionFailureCause), + httpStatus: Schema.optionalKey(Schema.Number), +}; + export class ConnectionTransientError extends Schema.TaggedError()( "ConnectionTransientError", { reason: ConnectionTransientReason, detail: Schema.String, traceId: Schema.optionalKey(Schema.String), + ...ConnectionFailureFields, }, ) { override get message(): string { @@ -95,6 +124,7 @@ export class ConnectionBlockedError extends Schema.TaggedError[0]) => - mapSessionRpcError(error, networkHint); + mapSessionRpcError(error, networkHint, via); yield* Effect.annotateCurrentSpan({ "connection.environment.id": connection.environmentId, }); @@ -185,6 +196,7 @@ export const make = Effect.fn("RpcSessionFactory.make")(function* ( ? `${connection.label} stopped responding.` : `${connection.label} disconnected.` }${networkHint}`, + ...socketFailureLocation(via), }), ), ), diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 7391bbaa31e41..99f999ba8935b 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -380,6 +380,24 @@ export const RelayEnvironmentEndpointUnavailableReason = Schema.Literals([ export type RelayEnvironmentEndpointUnavailableReason = typeof RelayEnvironmentEndpointUnavailableReason.Type; +/** + * Which part of the path from the relay to the environment failed. `tunnel_down` + * means Cloudflare has no live connector for the tunnel; `origin_unreachable` + * means the tunnel answered but T3 Code behind it did not. Optional on the wire + * so old relays and old clients keep working. + */ +export const RelayEnvironmentEndpointFailure = Schema.Literals([ + "tunnel_down", + "origin_unreachable", + "dns", + "tls", + "refused", + "timeout", + "invalid_response", + "unknown", +]); +export type RelayEnvironmentEndpointFailure = typeof RelayEnvironmentEndpointFailure.Type; + export const RelayAgentActivityPublishProofInvalidReason = Schema.Literals([ "invalid_signature_or_payload", "replayed_nonce", @@ -482,6 +500,8 @@ export class RelayEnvironmentEndpointUnavailableError extends Schema.TaggedError { code: Schema.Literal("environment_endpoint_unavailable"), reason: RelayEnvironmentEndpointUnavailableReason, + endpointFailure: Schema.optionalKey(RelayEnvironmentEndpointFailure), + endpointStatus: Schema.optionalKey(Schema.Number), traceId: TrimmedNonEmptyString, }, { httpApiStatus: 502 }, @@ -829,6 +849,8 @@ export const RelayEnvironmentStatusResponse = Schema.Struct({ checkedAt: TrimmedNonEmptyString, descriptor: Schema.optional(ExecutionEnvironmentDescriptor), error: Schema.optional(TrimmedNonEmptyString), + endpointFailure: Schema.optional(RelayEnvironmentEndpointFailure), + endpointStatus: Schema.optional(Schema.Number), offlineReason: Schema.optional(RelayEnvironmentOfflineReason), traceId: Schema.optional(TrimmedNonEmptyString), });