From b6edbd22d3b449be0b58b42054a644e42ef712c5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 3 Sep 2026 22:51:42 -0700 Subject: [PATCH 1/3] fix(connect): diagnose incomplete headless server setup --- apps/server/src/cli/connect.ts | 15 +- apps/server/src/cli/service.test.ts | 39 +++++- apps/server/src/cli/service.ts | 12 +- apps/server/src/cloud/bootService.test.ts | 144 ++++++++++++++++++-- apps/server/src/cloud/bootService.ts | 124 +++++++++++++++-- apps/server/src/cloud/http.ts | 12 +- apps/server/src/cloud/relayResponse.test.ts | 125 +++++++++++++++++ apps/server/src/cloud/relayResponse.ts | 82 +++++++++++ apps/server/src/server.ts | 11 +- docs/user/background-service.md | 36 ++++- docs/user/remote-access.md | 27 ++++ 11 files changed, 585 insertions(+), 42 deletions(-) create mode 100644 apps/server/src/cloud/relayResponse.test.ts create mode 100644 apps/server/src/cloud/relayResponse.ts diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 3f8e1d123da5..25cfb18f3402 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -35,6 +35,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BootService from "../cloud/bootService.ts"; import * as CliState from "../cloud/CliState.ts"; import * as CliTokenManager from "../cloud/CliTokenManager.ts"; +import { filterRelayResponse } from "../cloud/relayResponse.ts"; import { CLOUD_LINKED_USER_ID, isAgentActivityPublishingEnabledValue, @@ -208,6 +209,8 @@ function formatCloudStatus(status: CloudCliStatus, options?: { readonly json?: b ` Relay: ${status.relayUrl ?? "not provisioned"}`, ` Publish agent activity: ${status.publishAgentActivity ? "enabled" : "disabled"}`, ...formatRelayClientStatus(status.relayClient), + "", + "This is saved setup, not a live connection check. Check the background service with `t3 service status`.", ...(nextStep ? ["", `Next: ${nextStep}`] : []), ].join("\n"); } @@ -346,7 +349,7 @@ const unlinkRelayEnvironment = Effect.fn("cloud.cli.unlink_relay_environment")(f ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), httpClient.execute, - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(filterRelayResponse), Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), withRelayClientTracing, ); @@ -689,17 +692,17 @@ export const connectCommand = Command.make("connect", { // Show which account was linked so an unexpected identity (an // authorization code for a different account) is visible before the // machine is brought online. - yield* Console.log(`✓ Connected${connectedAs(linked.identity)}`); + yield* Console.log(`✓ Authorized${connectedAs(linked.identity)}`); - // Connect itself already succeeded; a boot-service failure must not - // fail the command, just tell the user what happened and move on. + // Authorization is stored. If service setup fails, preserve it and + // show how to run the server manually. const background = yield* recoverServiceOnboardingOffer(offerServiceDuringOnboarding); if (background) { const platform = yield* HostProcessPlatform; yield* Console.log( platform === "darwin" - ? "\n✓ Background service ready\n\nT3 Code will stay reachable while you are logged in to this Mac." - : "\n✓ Background service ready\n\nT3 Code will stay reachable after you log out.", + ? "\n✓ Background service ready\n\nT3 Code is set to run while you are logged in to this Mac. The server establishes the T3 Connect link on startup." + : "\n✓ Background service ready\n\nT3 Code is set to keep running after you log out. The server establishes the T3 Connect link on startup.", ); return; } diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index b442d46df368..f1eca7d451ca 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -45,10 +45,29 @@ it("reports the installed service version and host paths", () => { it("gives a direct repair command for a stale service", () => { assert.include( formatServiceStatus({ ...status, current: false }, "0.0.29"), - "Next: Run `npx t3@latest service update`.", + "Next: Run `npx t3@0.0.29 service update`.", ); }); +it("explains an incomplete nightly installation and keeps repair on its installed version", () => { + const output = formatServiceStatus( + { + ...status, + current: false, + installedVersion: "0.0.32-nightly.1", + problems: ["linger-disabled", "service-stopped"], + }, + "0.0.32-nightly.1", + ); + + expect(output).toContain("[linger-disabled]"); + expect(output).toContain("last login session ends"); + expect(output).toContain('sudo loginctl enable-linger "$(id -un)"'); + expect(output).toContain("[service-stopped]"); + expect(output).toContain("npx t3@0.0.32-nightly.1 service update"); + expect(output).not.toContain("t3@latest"); +}); + it("explains where the service is supported", () => { assert.include( formatServiceStatus({ ...status, supported: false, installed: false }, "0.0.29"), @@ -151,6 +170,15 @@ it.effect.each([ name: "the same version", state: { ...status, current: false, installedVersion: packageJson.version }, }, + { + name: "an incomplete install of the same version", + state: { + ...status, + current: false, + installedVersion: packageJson.version, + problems: ["linger-disabled"] as const, + }, + }, { name: "an unknown version", state: { ...status, current: false } }, ])("installs or repairs $name without an override", ({ state }) => Effect.gen(function* () { @@ -201,3 +229,12 @@ it.effect("keeps onboarding successful when a newer version appears before insta expect(ready).toBe(false); }), ); + +it.effect("keeps the manual-server fallback when background prerequisites fail", () => + Effect.gen(function* () { + const ready = yield* recoverServiceOnboardingOffer( + Effect.fail(new BootService.BootServicePrerequisiteError({ problem: "linger-disabled" })), + ); + expect(ready).toBe(false); + }), +); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index 3e2e893334f0..e59611a9c3e3 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -68,6 +68,9 @@ export function formatServiceStatus( return "T3 Code service\n Status: not installed\n Next: Run `t3 service install`."; } const installedVersion = status.installedVersion ?? cliVersion; + const problems = (status.problems ?? []).map( + (problem) => ` [${problem}] ${BootService.formatBootServiceProblem(problem)}`, + ); if ( !status.current && status.installedVersion !== undefined && @@ -78,6 +81,7 @@ export function formatServiceStatus( ` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, + ...problems, ` Next: Use \`npx t3@${installedVersion} service update\` to repair it, or pass \`--allow-downgrade\` explicitly.`, ].join("\n"); } @@ -86,7 +90,8 @@ export function formatServiceStatus( ` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`, ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, - ...(status.current ? [] : [" Next: Run `npx t3@latest service update`."]), + ...problems, + ...(status.current ? [] : [` Next: Run \`npx t3@${installedVersion} service update\`.`]), ].join("\n"); } @@ -189,6 +194,9 @@ export const offerServiceDuringOnboarding = Effect.gen(function* () { yield* Console.log("T3 Code is already set up to run in the background on this machine."); return true; } + for (const problem of status.problems ?? []) { + yield* Console.warn(`[${problem}] ${BootService.formatBootServiceProblem(problem)}`); + } if ( installed && status.installedVersion !== undefined && @@ -239,6 +247,8 @@ export const recoverServiceOnboardingOffer = ( Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceInstallError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), + BootServicePrerequisiteError: (error) => + Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceUpdatePendingError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceDowngradeRefusedError: (error) => diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 3e31ab3f6371..c820f93cd212 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -127,8 +127,17 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const commands: string[] = []; const timeouts = new Map(); - const control: { failCommand: string | undefined; stateAfterStop?: string } = { + const control: { + failCommand: string | undefined; + stateAfterStop?: string; + linger: string; + enabled: boolean; + active: boolean; + } = { failCommand: undefined, + linger: "yes", + enabled: true, + active: true, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -137,6 +146,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const command = `${input.command} ${input.args.join(" ")}`; commands.push(command); timeouts.set(command, input.timeout); + const failed = command === control.failCommand; + if (!failed && command === "loginctl enable-linger --no-ask-password 501") + control.linger = "yes"; + if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; + if (!failed && command === "systemctl --user restart t3code.service") control.active = true; if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || @@ -145,9 +159,20 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( yield* fs.writeFileString(statePath, control.stateAfterStop).pipe(Effect.orDie); } return { - stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "", + stdout: + input.args[1] === "--version" + ? "t3 v1.2.3\n" + : input.command === "loginctl" && input.args[0] === "show-user" + ? `${control.linger}\n` + : input.args[1] === "is-enabled" + ? control.enabled + ? "enabled\n" + : "disabled\n" + : "", stderr: "", - code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), + code: ChildProcessSpawner.ExitCode( + failed || (input.args[1] === "is-active" && !control.active) ? 1 : 0, + ), timedOut: false, stdoutTruncated: false, stderrTruncated: false, @@ -182,10 +207,103 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( ), ); const service = yield* makeService(); - return { service, makeService, fs, statePath, commands, timeouts, control }; + return { service, makeService, fs, statePath, commands, timeouts, control, runtime }; }); it.layer(NodeServices.layer)("boot service install", (it) => { + it.effect( + "fails before installing files or validating a runtime when lingering needs an administrator", + () => + Effect.gen(function* () { + const { service, fs, statePath, commands, control, runtime } = yield* makeHarness(); + const before = yield* service.status; + control.linger = "no"; + control.failCommand = "loginctl enable-linger --no-ask-password 501"; + yield* fs.remove(runtime.sentinelPath); + + const error = yield* service.install().pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "BootServicePrerequisiteError", + problem: "linger-disabled", + }); + expect(error.message).toContain('sudo loginctl enable-linger "$(id -un)"'); + expect(error.message).toContain("last login session ends"); + expect(yield* fs.exists(before.unitPath)).toBe(false); + expect(yield* fs.exists(statePath)).toBe(false); + expect( + commands.some((command) => command.startsWith("npm ") || command.includes("--version")), + ).toBe(false); + expect( + commands.some( + (command) => command.includes("daemon-reload") || command.includes("restart"), + ), + ).toBe(false); + expect(yield* fs.readFileString(before.logPath)).toContain("[linger-disabled]"); + }), + ); + + it.effect( + "detects a partial install and preserves the running service when repair lacks permission", + () => + Effect.gen(function* () { + const { service, fs, statePath, commands, control } = yield* makeHarness(); + const plan = yield* service.install(); + const before = yield* fs.readFileString(statePath); + const unit = yield* fs.readFileString(plan.unitPath); + control.linger = "no"; + control.failCommand = "loginctl enable-linger --no-ask-password 501"; + + expect(yield* service.status).toMatchObject({ + current: false, + problems: ["linger-disabled"], + }); + commands.length = 0; + expect((yield* service.install().pipe(Effect.flip))._tag).toBe( + "BootServicePrerequisiteError", + ); + expect(yield* fs.readFileString(statePath)).toBe(before); + expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); + expect(commands).not.toContain("systemctl --user stop t3code.service"); + }), + ); + + it.effect("enables lingering before installing and repairs stopped or disabled services", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + control.linger = "no"; + yield* service.install(); + expect(control.linger).toBe("yes"); + expect(commands.indexOf("loginctl enable-linger --no-ask-password 501")).toBeLessThan( + commands.indexOf("systemctl --user daemon-reload"), + ); + + control.enabled = false; + control.active = false; + expect(yield* service.status).toMatchObject({ + current: false, + problems: ["service-disabled", "service-stopped"], + }); + yield* service.install(); + expect((yield* service.status).current).toBe(true); + }), + ); + + it.effect.each([ + { command: "systemctl --user show-environment", problem: "user-manager-unavailable" }, + { command: "loginctl show-user 501 --property=Linger --value", problem: "linger-unavailable" }, + ])("reports failed prerequisite probes without installing: $command", ({ command, problem }) => + Effect.gen(function* () { + const { service, fs, statePath, control } = yield* makeHarness(); + control.failCommand = command; + expect(yield* service.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServicePrerequisiteError", + problem, + }); + expect(yield* fs.exists(statePath)).toBe(false); + }), + ); + it.effect("installs, reports current state, and uninstalls", () => Effect.gen(function* () { const { service, fs, statePath, commands, timeouts } = yield* makeHarness(); @@ -286,8 +404,10 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect(yield* fs.readFileString(plan.launcherPath)).toBe(launcher); expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); expect( - commands.filter((command) => - command.startsWith(platform === "linux" ? "systemctl " : "launchctl "), + commands.filter( + (command) => + command.startsWith(platform === "linux" ? "systemctl " : "launchctl ") && + !command.includes("show-environment"), ), ).toEqual( platform === "linux" @@ -351,7 +471,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const error = yield* service.install().pipe(Effect.flip); expect(error._tag).toBe("BootServiceCommandError"); - expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ + expect( + commands.filter( + (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), + ), + ).toEqual([ "systemctl --user stop t3code.service", "systemctl --user daemon-reload", "systemctl --user restart t3code.service", @@ -382,7 +506,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => { "BootServiceUpdatePendingError", ); expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true); - expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ + expect( + commands.filter( + (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), + ), + ).toEqual([ "systemctl --user stop t3code.service", "systemctl --user restart t3code.service", ]); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 59530498076e..50ffcedf74b8 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -235,7 +235,6 @@ function systemdManager(input: { command: "systemctl", args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE], }, - { step: "enabling lingering for this user", command: "loginctl", args: ["enable-linger"] }, // Start last. No administrative state write occurs after this succeeds. { step: "starting the service", @@ -409,6 +408,40 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass()( + "BootServicePrerequisiteError", + { problem: BootServiceProblem, cause: Schema.optional(Schema.Defect()) }, +) { + override get message(): string { + return `[${this.problem}] ${formatBootServiceProblem(this.problem)}`; + } +} + export class BootServiceUpdatePendingError extends Schema.TaggedErrorClass()( "BootServiceUpdatePendingError", {}, @@ -434,6 +467,7 @@ export type BootServiceError = | BootServiceUnsupportedError | BootServiceCommandError | BootServiceInstallError + | BootServicePrerequisiteError | BootServiceUpdatePendingError | BootServiceDowngradeRefusedError; @@ -442,6 +476,7 @@ export interface BootServiceStatus { readonly installed: boolean; readonly current: boolean; readonly installedVersion?: string; + readonly problems?: ReadonlyArray; readonly unitPath: string; readonly logPath: string; } @@ -539,6 +574,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { : Effect.succeed(detectedManager), ); + const logFailure = (error: { readonly message: string }) => + DateTime.now.pipe( + Effect.flatMap((now) => + fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { flag: "a" }), + ), + Effect.ignore, + ); + const runStep = Effect.fn("cloud.boot_service.run_step")(function* ( step: string, command: string, @@ -557,16 +600,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { stderrLength: result.stderr.length, }), ), - Effect.tapError((error) => - DateTime.now.pipe( - Effect.flatMap((now) => - fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { - flag: "a", - }), - ), - Effect.ignore, - ), - ), + Effect.tapError(logFailure), ); }); @@ -587,6 +621,66 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { { discard: true }, ); + const probe = (command: string, args: ReadonlyArray) => + runner.run({ command, args, timeout: Duration.seconds(5) }).pipe(Effect.option); + const succeeded = (result: Option.Option) => + Option.isSome(result) && result.value.code === 0; + const lingerArgs = [ + "show-user", + ...(uid === undefined ? [] : [String(uid)]), + "--property=Linger", + "--value", + ]; + const readSystemdProblems = Effect.fn("cloud.boot_service.read_systemd_problems")(function* ( + includeService: boolean, + ) { + const [manager, linger] = yield* Effect.all( + [probe("systemctl", ["--user", "show-environment"]), probe("loginctl", lingerArgs)], + { concurrency: "unbounded" }, + ); + const problems: BootServiceProblem[] = []; + if (!succeeded(manager)) problems.push("user-manager-unavailable"); + const lingering = succeeded(linger) && Option.isSome(linger) ? linger.value.stdout.trim() : ""; + if (lingering !== "yes") { + problems.push(lingering === "no" ? "linger-disabled" : "linger-unavailable"); + } + if (includeService && succeeded(manager)) { + const [enabled, active] = yield* Effect.all( + [ + probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]), + probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]), + ], + { concurrency: "unbounded" }, + ); + if ( + !succeeded(enabled) || + (Option.isSome(enabled) && enabled.value.stdout.trim() !== "enabled") + ) { + problems.push("service-disabled"); + } + if (!succeeded(active)) problems.push("service-stopped"); + } + return problems; + }); + + const requireSystemdPrerequisites = Effect.gen(function* () { + const problems = yield* readSystemdProblems(false); + const unavailable = problems.find((problem) => problem !== "linger-disabled"); + if (unavailable) return yield* new BootServicePrerequisiteError({ problem: unavailable }); + if (!problems.includes("linger-disabled")) return; + yield* runStep("enabling lingering for this user", "loginctl", [ + "enable-linger", + "--no-ask-password", + ...(uid === undefined ? [] : [String(uid)]), + ]).pipe( + Effect.mapError( + (cause) => new BootServicePrerequisiteError({ problem: "linger-disabled", cause }), + ), + ); + const remaining = yield* readSystemdProblems(false); + if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] }); + }); + const install = Effect.fn("cloud.boot_service.install")(function* (options?: { readonly allowDowngrade?: boolean; }) { @@ -595,6 +689,11 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { .makeDirectory(input.logsDir, { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + // A permissions failure must not leave a partial install or stop a working server. + if (manager.kind === "systemd") { + yield* requireSystemdPrerequisites.pipe(Effect.tapError(logFailure)); + } + // Prepare every immutable artifact before stopping the installed unit. yield* ensurePinnedRuntimeInstalled({ baseDir: input.baseDir, @@ -743,11 +842,14 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { detectedManager.kind === "launchd" ? contents.replace(/(PATH<\/key>\n\s*)[^<]*(<\/string>)/, "$1$2") : contents; + const problems = detectedManager.kind === "systemd" ? yield* readSystemdProblems(true) : []; return { supported: true, installed: true, ...(installedVersion === undefined ? {} : { installedVersion }), + problems, current: + problems.length === 0 && normalizeUnit(unit) === normalizeUnit(detectedManager.render(plan)) && launcherExists && runtimeEntryExists && diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 29fdfe8ece2f..e0d458b4b97c 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -85,6 +85,7 @@ import { import * as CliTokenManager from "./CliTokenManager.ts"; import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts"; import { traceRelayRequest } from "./traceRelayRequest.ts"; +import { filterRelayResponse, relayRequestError } from "./relayResponse.ts"; const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; @@ -526,14 +527,9 @@ const relayClientRequest = ( HttpClientRequest.bearerToken(input.token), HttpClientRequest.bodyJson(input.payload), Effect.flatMap(dependencies.httpClient.execute), - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(filterRelayResponse), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), - Effect.mapError( - (cause) => - new EnvironmentHttpInternalServerError({ - message: `T3 Connect relay request failed: ${String(cause)}`, - }), - ), + Effect.mapError(relayRequestError), withRelayClientTracing, ); @@ -720,7 +716,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), dependencies.httpClient.execute, - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(filterRelayResponse), Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), withRelayClientTracing, ); diff --git a/apps/server/src/cloud/relayResponse.test.ts b/apps/server/src/cloud/relayResponse.test.ts new file mode 100644 index 000000000000..40325a9b9192 --- /dev/null +++ b/apps/server/src/cloud/relayResponse.test.ts @@ -0,0 +1,125 @@ +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts"; + +const response = ( + status: number, + body: string | Record, + headers?: Record, +) => + HttpClientResponse.fromWeb( + HttpClientRequest.post("https://relay.example.test/v1/client/environment-links"), + typeof body === "string" + ? new Response(body, { status, ...(headers ? { headers } : {}) }) + : Response.json(body, { status, ...(headers ? { headers } : {}) }), + ); + +it.effect("reports the tunnel limit and relay trace instead of a generic 403", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response(403, { + _tag: "RelayEnvironmentLinkLimitExceededError", + code: "environment_link_limit_exceeded", + maxTunnels: 3, + traceId: "trace-limit", + }), + ).pipe(Effect.mapError(relayRequestError), Effect.flip); + + expect(error._tag).toBe("EnvironmentHttpForbiddenError"); + expect(error.message).toContain("at most 3 tunnels"); + expect(error.message).toContain("Unlink an unused environment"); + expect(error.message).toContain("Trace ID: trace-limit"); + }), +); + +it.effect("makes revoked authorization actionable and non-retryable", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response(401, { + _tag: "RelayAuthInvalidError", + code: "auth_invalid", + reason: "invalid_bearer", + traceId: "trace-auth", + }), + ).pipe(Effect.mapError(relayRequestError), Effect.flip); + + expect(error._tag).toBe("EnvironmentHttpUnauthorizedError"); + expect(error.message).toContain("invalid_bearer"); + expect(error.message).toContain("t3 connect login"); + expect(error.message).toContain("Trace ID: trace-auth"); + }), +); + +it.effect("reports an unrecognized access denial without printing its response body", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response(403, "private upstream details", { + "content-type": "text/html", + "cf-ray": "abcdef1234-IAD", + }), + ).pipe(Effect.flip); + + expect(error._tag).toBe("EnvironmentHttpForbiddenError"); + expect(error.message).toContain("HTTP 403"); + expect(error.message).toContain("proxy or firewall"); + expect(error.message).toContain("Cloudflare Ray ID: abcdef1234-IAD"); + expect(error.message).not.toContain("private upstream details"); + }), +); + +it.effect.each([408, 429, 500, 502, 503, 504])( + "keeps transient HTTP %s failures retryable", + (status) => + Effect.gen(function* () { + const error = yield* filterRelayResponse(response(status, "unavailable")).pipe(Effect.flip); + expect(error._tag).toBe("EnvironmentHttpInternalServerError"); + expect(error.message).toContain(`HTTP ${status}`); + }), +); + +it.effect.each([ + { status: 401, attempts: 1 }, + { status: 403, attempts: 1 }, + { status: 429, attempts: 2 }, + { status: 503, attempts: 2 }, +])("stops rejected startup links but retries temporary failures: $status", ({ status, attempts }) => + Effect.gen(function* () { + let requests = 0; + const result = yield* Effect.suspend(() => { + requests++; + return filterRelayResponse(response(requests === 1 ? status : 200, "{}")); + }).pipe( + Effect.mapError(relayRequestError), + Effect.retry({ while: shouldRetryCloudLink, times: 1 }), + Effect.result, + ); + expect(requests).toBe(attempts); + expect(result._tag).toBe(attempts === 1 ? "Failure" : "Success"); + }), +); + +it.effect("keeps the relay failure reason and trace when tunnel cleanup fails", () => + Effect.gen(function* () { + const error = yield* filterRelayResponse( + response(500, { + _tag: "RelayInternalError", + code: "internal_error", + reason: "upstream_unavailable", + traceId: "trace-cleanup", + }), + ).pipe(Effect.flip); + + expect(error._tag).toBe("EnvironmentHttpInternalServerError"); + expect(error.message).toContain("upstream_unavailable"); + expect(error.message).toContain("Trace ID: trace-cleanup"); + }), +); + +it.effect("leaves successful response bodies available to their decoder", () => + Effect.gen(function* () { + const result = yield* filterRelayResponse(response(200, '{"ok":true}')); + expect(yield* result.json).toEqual({ ok: true }); + }), +); diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts new file mode 100644 index 000000000000..451e97290ea1 --- /dev/null +++ b/apps/server/src/cloud/relayResponse.ts @@ -0,0 +1,82 @@ +import { + EnvironmentHttpBadRequestError, + EnvironmentHttpConflictError, + EnvironmentHttpForbiddenError, + EnvironmentHttpInternalServerError, + EnvironmentHttpUnauthorizedError, +} from "@t3tools/contracts"; +import { RelayProtectedError } from "@t3tools/contracts/relay"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; +import { isHttpClientError } from "effect/unstable/http/HttpClientError"; + +const isRelayResponseError = Schema.is( + Schema.Union([ + EnvironmentHttpBadRequestError, + EnvironmentHttpForbiddenError, + EnvironmentHttpInternalServerError, + EnvironmentHttpUnauthorizedError, + ]), +); + +export function relayRequestError(cause: unknown) { + return isRelayResponseError(cause) + ? cause + : new EnvironmentHttpInternalServerError({ + message: `Could not complete the T3 Connect relay request. ${isHttpClientError(cause) ? cause.message : "The relay returned an unexpected response."} Check this machine's network connection and relay availability, then retry.`, + }); +} + +const isPermanentCloudLinkError = Schema.is( + Schema.Union([ + EnvironmentHttpBadRequestError, + EnvironmentHttpForbiddenError, + EnvironmentHttpUnauthorizedError, + EnvironmentHttpConflictError, + ]), +); + +export const shouldRetryCloudLink = (error: unknown): boolean => !isPermanentCloudLinkError(error); + +function recoveryHint(error: RelayProtectedError): string { + switch (error._tag) { + case "RelayEnvironmentLinkLimitExceededError": + return "Unlink an unused environment in T3 Connect, then restart T3 Code on this machine."; + case "RelayAuthInvalidError": + return "Run `t3 connect login` to check this machine's authorization. If the stored credential was revoked, sign out with `t3 connect logout`, then run `t3 connect` again. Restart T3 Code after signing in."; + case "RelayEnvironmentLinkProofExpiredError": + case "RelayEnvironmentLinkProofInvalidError": + return "Check this machine's date and time, update T3 Code, then restart it."; + default: + return "Retry when the relay is available. If this continues, include the trace ID when reporting it."; + } +} + +/** Preserve relay diagnostics before converting permanent rejections into non-retryable errors. */ +export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(function* ( + response: HttpClientResponse.HttpClientResponse, +) { + if (response.status >= 200 && response.status < 300) return response; + const decoded = yield* HttpClientResponse.schemaBodyJson(RelayProtectedError)(response).pipe( + Effect.option, + ); + const ray = response.headers["cf-ray"]; + const requestId = ray && /^[a-zA-Z0-9-]{1,128}$/.test(ray) ? ` Cloudflare Ray ID: ${ray}.` : ""; + const message = Option.isSome(decoded) + ? `T3 Connect: ${decoded.value.message}. ${recoveryHint(decoded.value)} Trace ID: ${decoded.value.traceId}.` + : `T3 Connect relay returned HTTP ${response.status} without a recognized error response. Check relay access and any proxy or firewall restrictions, then restart T3 Code.${requestId}`; + + if (response.status === 401) return yield* new EnvironmentHttpUnauthorizedError({ message }); + if (response.status === 403) return yield* new EnvironmentHttpForbiddenError({ message }); + if ( + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 + ) { + return yield* new EnvironmentHttpBadRequestError({ message }); + } + return yield* new EnvironmentHttpInternalServerError({ message }); +}); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0e02bf464fa7..c0be0c444573 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,4 +1,5 @@ import { EnvironmentHttpApi, ProviderDriverKind } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -106,6 +107,7 @@ import { releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; +import { shouldRetryCloudLink } from "./cloud/relayResponse.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; @@ -675,7 +677,7 @@ export const makeServerLayer = Layer.unwrap( Effect.catchCause((cause) => Effect.logWarning( "Failed to release the managed tunnel on shutdown; the next link reuses it", - { cause }, + { errors: Cause.prettyErrors(cause).map((error) => error.message) }, ), ), Effect.asVoid, @@ -707,10 +709,7 @@ export const makeServerLayer = Layer.unwrap( // reachability after a restart. yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", + while: shouldRetryCloudLink, schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), @@ -721,7 +720,7 @@ export const makeServerLayer = Layer.unwrap( Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), Effect.catch((cause) => Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, + message: cause.message, }), ), ); diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 23ac07cee0db..6ecf2ec52c81 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -11,7 +11,8 @@ Install it with the latest T3 Code release: npx t3@latest service install ``` -Check whether it is installed: +Check whether it is installed. On Linux this also checks whether the service is running, enabled +at startup, and allowed to keep running after logout: ```sh npx t3@latest service status @@ -58,6 +59,8 @@ updates roll back with the server version. An older launcher may require one loc **Linux** uses a systemd user unit at `~/.config/systemd/user/t3code.service`. The service starts when the machine boots and keeps running after you log out (lingering is enabled during install). +Setup checks the systemd user manager and enables lingering before installing a runtime or stopping +an existing service. If that requires administrator permission, setup stops with a recovery command. **macOS** uses a launch agent at `~/Library/LaunchAgents/com.t3tools.t3code.service.plist`. It starts when you log in, not when the Mac boots, and it stops when you log out; macOS has no @@ -87,3 +90,34 @@ background. This is only an onboarding shortcut: the service and T3 Connect are Signing out of T3 Connect does not remove the service. Use `t3 service uninstall` when you no longer want T3 Code to start in the background. + +## Troubleshooting + +Run `t3 service status` on the server machine. An installed version alone does not mean the service +is running or will survive logout. Linux status reports these problems: + +| Code | What it means | Recovery | +| -------------------------- | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `linger-disabled` | The service stops after your last login session ends and does not start at boot. | Run `sudo loginctl enable-linger "$(id -un)"`, then retry setup as your normal user. | +| `linger-unavailable` | T3 Code could not verify the logout setting. | Run `loginctl show-user "$(id -un)" --property=Linger` and check that systemd-logind is available. | +| `user-manager-unavailable` | T3 Code cannot reach your systemd user manager. | Run `systemctl --user status` in a login session for the service user. Install your distribution's systemd user-session support if needed. | +| `service-disabled` | The service is not enabled to start automatically. | Run the repair command shown by `t3 service status`. | +| `service-stopped` | The service is installed but is not running. | Read the service log and `systemctl --user status t3code.service`, then run the displayed repair command. | + +For an SSH host, run the administrator command in an interactive terminal so sudo can prompt for +your password: + +```sh +ssh -t your-server 'sudo loginctl enable-linger "$(id -un)"' +``` + +Run only the `loginctl` command with sudo. Running `t3` with sudo creates a separate installation and +Connect identity for root. If an administrator is unavailable, run `t3 serve` in a terminal and +keep that session open. + +The repair command shown by status uses the installed version, including its nightly version. +This repairs an incomplete install without accidentally switching release channels. Setup leaves an +existing service running if the user-manager or lingering check fails. + +`t3 service status` prints the log path. The adjacent `server.trace.ndjson` file contains detailed +server traces. For failures after authorization, see [T3 Connect troubleshooting](./remote-access.md#t3-connect-troubleshooting). diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 17e60e8ee4ce..50a07b50fd2b 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -2,6 +2,33 @@ Use this when you want to connect to a T3 Code server from another device such as a phone, tablet, or separate desktop app. +## T3 Connect troubleshooting + +Run `t3 connect` on the server machine to authorize it and optionally install the background service. +The authorization message means your sign-in was saved. The server must then start and establish its +relay link before the machine is reachable. + +`t3 connect status` reports saved authorization and link configuration, not a live reachability +check. If the machine appears offline, run `t3 service status` on it and read the displayed log. +On Linux, a service that works while SSH is open but stops after logout usually has lingering +disabled. See [background service troubleshooting](./background-service.md#troubleshooting). + +Relay errors include the returned reason and trace ID when available: + +| Error | Next step | +| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `environment_link_limit_exceeded` / managed tunnel limit reached | Unlink an unused environment in T3 Connect, then restart T3 Code on this machine. | +| `auth_invalid` / `invalid_bearer` | Run `t3 connect login`. If the stored credential was revoked, run `t3 connect logout`, then `t3 connect` and restart the server. | +| Expired or invalid link proof | Check the server's date and time, update T3 Code, and restart it. Include the reason and trace ID if it still fails. | +| HTTP 403 without a recognized error response | Check relay access and any proxy or firewall restrictions. Include the Cloudflare Ray ID if one was returned; an HTTP status alone does not identify the cause. | +| HTTP 408, 429, or 5xx | The server retries temporary failures during startup for up to ten minutes. Check network and relay availability; include the trace ID when reporting a persistent failure. | + +Authorization and other permanent 4xx rejections stop the startup link attempt immediately. +After correcting them, restart the server. For the Linux background service, use +`systemctl --user restart t3code.service`; for a foreground server, stop it and run `t3 serve` again. +Keep the diagnostic message and trace ID when reporting a problem. Do not post authorization codes, +pairing URLs, or the contents of the secrets directory. + ## Quick Pairing for a Running Server If a server is already running on this machine, mint a fresh pairing token and QR code without restarting anything: From 582fca96a39555b2c0747439a3eae3979dc164a9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 3 Sep 2026 22:53:33 -0700 Subject: [PATCH 2/3] fix(connect): keep repair suggestions on the newer version --- apps/server/src/cli/service.test.ts | 9 +++++++++ apps/server/src/cli/service.ts | 2 +- docs/user/background-service.md | 6 +++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index f1eca7d451ca..38732e42987a 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -68,6 +68,15 @@ it("explains an incomplete nightly installation and keeps repair on its installe expect(output).not.toContain("t3@latest"); }); +it("suggests the newer CLI version when the installed service needs an update", () => { + const output = formatServiceStatus( + { ...status, current: false, installedVersion: "0.0.28" }, + "0.0.29", + ); + expect(output).toContain("npx t3@0.0.29 service update"); + expect(output).not.toContain("npx t3@0.0.28 service update"); +}); + it("explains where the service is supported", () => { assert.include( formatServiceStatus({ ...status, supported: false, installed: false }, "0.0.29"), diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index e59611a9c3e3..0cea18ff4977 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -91,7 +91,7 @@ export function formatServiceStatus( ` Unit: ${status.unitPath}`, ` Logs: ${status.logPath}`, ...problems, - ...(status.current ? [] : [` Next: Run \`npx t3@${installedVersion} service update\`.`]), + ...(status.current ? [] : [` Next: Run \`npx t3@${cliVersion} service update\`.`]), ].join("\n"); } diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 6ecf2ec52c81..a89fc2f7842d 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -115,9 +115,9 @@ Run only the `loginctl` command with sudo. Running `t3` with sudo creates a sepa Connect identity for root. If an administrator is unavailable, run `t3 serve` in a terminal and keep that session open. -The repair command shown by status uses the installed version, including its nightly version. -This repairs an incomplete install without accidentally switching release channels. Setup leaves an -existing service running if the user-manager or lingering check fails. +The repair command shown by status uses the CLI version, or the installed service version if that +is newer. An older stable CLI therefore does not recommend downgrading a nightly installation. +Setup leaves an existing service running if the user-manager or lingering check fails. `t3 service status` prints the log path. The adjacent `server.trace.ndjson` file contains detailed server traces. For failures after authorization, see [T3 Connect troubleshooting](./remote-access.md#t3-connect-troubleshooting). From 5aaf56c9491fa672e0b58ae5b58804cc71659638 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 3 Sep 2026 22:59:06 -0700 Subject: [PATCH 3/3] fix(connect): bound transport failure diagnostics --- apps/server/src/cloud/relayResponse.test.ts | 20 ++++++++++++++++++++ apps/server/src/cloud/relayResponse.ts | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cloud/relayResponse.test.ts b/apps/server/src/cloud/relayResponse.test.ts index 40325a9b9192..9f53378f14ce 100644 --- a/apps/server/src/cloud/relayResponse.test.ts +++ b/apps/server/src/cloud/relayResponse.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import { HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import * as HttpClientError from "effect/unstable/http/HttpClientError"; import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts"; @@ -16,6 +17,25 @@ const response = ( : Response.json(body, { status, ...(headers ? { headers } : {}) }), ); +it("reports a transport failure category without exposing request or cause details", () => { + const error = relayRequestError( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.post("https://relay.example.test/link?token=private-token"), + description: "private transport details", + cause: new Error("private cause details"), + }), + }), + ); + + expect(error._tag).toBe("EnvironmentHttpInternalServerError"); + expect(error.message).toContain("TransportError"); + expect(error.message).toContain("network connection"); + expect(error.message).not.toContain("relay.example.test"); + expect(error.message).not.toContain("private"); + expect(shouldRetryCloudLink(error)).toBe(true); +}); + it.effect("reports the tunnel limit and relay trace instead of a generic 403", () => Effect.gen(function* () { const error = yield* filterRelayResponse( diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index 451e97290ea1..df04642158eb 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -25,7 +25,7 @@ export function relayRequestError(cause: unknown) { return isRelayResponseError(cause) ? cause : new EnvironmentHttpInternalServerError({ - message: `Could not complete the T3 Connect relay request. ${isHttpClientError(cause) ? cause.message : "The relay returned an unexpected response."} Check this machine's network connection and relay availability, then retry.`, + message: `Could not complete the T3 Connect relay request. ${isHttpClientError(cause) ? `The relay request failed (${cause.reason._tag}).` : "The relay returned an unexpected response."} Check this machine's network connection and relay availability, then retry.`, }); }