fix: smooth remote server updates

apps/server/src/cloud/http.test.ts

@@ -1,7 +1,10 @@
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { describe, expect, it } from "@effect/vitest";
 import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
+import * as Path from "effect/Path";
 import * as PlatformError from "effect/PlatformError";
 import * as Tracer from "effect/Tracer";
 import {
@@ -15,6 +18,14 @@ import { EnvironmentId } from "@t3tools/contracts";
 import { RelayClientTracer } from "@t3tools/shared/relayTracing";
 import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ServerConfigModule from "../config.ts";
+import { writeServiceState } from "../serviceLauncher.ts";
+import {
+  SERVICE_LAUNCHER_PROTOCOL,
+  SERVICE_STATE_FILE,
+  SERVICE_STOP_MARKER_FILE,
+  type ServiceUpdateRecord,
+} from "./serviceProtocol.ts";
 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
 import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
 import * as CliTokenManager from "./CliTokenManager.ts";
@@ -256,6 +267,23 @@ describe("releaseManagedTunnelOnShutdown", () => {
     readonly respond?: () => Response;
   }
 
+  // Writes the launcher's durable state file into this test's baseDir with
+  // the launcher's own writer; the release reads it to detect an in-flight
+  // update handoff.
+  const writeLauncherState = (update: ServiceUpdateRecord) =>
+    Effect.gen(function* () {
+      const path = yield* Path.Path;
+      const config = yield* ServerConfigModule.ServerConfig;
+      const statePath = path.join(config.baseDir, "runtime", SERVICE_STATE_FILE);
+      yield* Effect.promise(() =>
+        writeServiceState(statePath, {
+          protocol: SERVICE_LAUNCHER_PROTOCOL,
+          activeVersion: "0.0.30",
+          update,
+        }),
+      );
+    });
+
   const provideReleaseHarness =
     (harness: ReleaseHarness) =>
     <A, E, R>(effect: Effect.Effect<A, E, R>) =>
@@ -306,6 +334,14 @@ describe("releaseManagedTunnelOnShutdown", () => {
             }),
           ),
         ),
+        // The release consults the launcher state file under the configured
+        // baseDir, so every harness run gets a scoped temp baseDir.
+        Effect.provide(
+          ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" }).pipe(
+            Layer.provideMerge(NodeServices.layer),
+          ),
+        ),
+        Effect.scoped,
       );
 
   // The persisted state of a CLI-managed link whose tunnel is releasable.
@@ -390,6 +426,83 @@ describe("releaseManagedTunnelOnShutdown", () => {
     }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
   });
 
+  it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
+    const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+    const applyConfigCalls: Array<unknown> = [];
+    const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+    return Effect.gen(function* () {
+      yield* writeLauncherState({
+        id: "update-1",
+        fromVersion: "0.0.30",
+        targetVersion: "0.0.31",
+        dbPath: "/tmp/state.sqlite",
+        status: "pending",
+      });
+
+      const released = yield* releaseManagedTunnelOnShutdown();
+
+      // The launcher restarts a server immediately, so the tunnel is not
+      // orphaned; keeping it avoids the hostname route re-propagation that
+      // dominates update downtime. The stored config must survive so the
+      // next boot respawns the connector against the same tunnel.
+      expect(released).toBe(false);
+      expect(applyConfigCalls).toEqual([]);
+      expect(requests).toEqual([]);
+      expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
+    }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+  });
+
+  it.effect("still releases a pending update when the launcher is stopping", () => {
+    // `t3 service uninstall` or `systemctl stop` during the pending window:
+    // the launcher writes its stop marker before signalling the child, so no
+    // replacement server is coming and the tunnel must not be kept.
+    const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+    const applyConfigCalls: Array<unknown> = [];
+    const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+    return Effect.gen(function* () {
+      yield* writeLauncherState({
+        id: "update-1",
+        fromVersion: "0.0.30",
+        targetVersion: "0.0.31",
+        dbPath: "/tmp/state.sqlite",
+        status: "pending",
+      });
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const config = yield* ServerConfigModule.ServerConfig;
+      yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
+
+      const released = yield* releaseManagedTunnelOnShutdown();
+
+      expect(released).toBe(true);
+      expect(requests).toHaveLength(1);
+      expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
+    }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+  });
+
+  it.effect("still releases when the recorded update already settled", () => {
+    const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+    const applyConfigCalls: Array<unknown> = [];
+    const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+    return Effect.gen(function* () {
+      yield* writeLauncherState({
+        id: "update-1",
+        fromVersion: "0.0.30",
+        targetVersion: "0.0.31",
+        status: "committed",
+      });
+
+      const released = yield* releaseManagedTunnelOnShutdown();
+
+      expect(released).toBe(true);
+      expect(requests).toHaveLength(1);
+      expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
+    }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+  });
+
   it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
     const { store, values } = makeMemorySecretStore(managedLinkSecrets);
     const applyConfigCalls: Array<unknown> = [];

apps/server/src/cloud/http.ts

@@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime";
 import * as Crypto from "effect/Crypto";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
 import * as Option from "effect/Option";
+import * as Path from "effect/Path";
 import * as Schema from "effect/Schema";
 import * as HttpEffect from "effect/unstable/http/HttpEffect";
 import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
@@ -56,8 +58,14 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
 import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
 import { requireEnvironmentScope } from "../auth/http.ts";
+import * as ServerConfig from "../config.ts";
 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
 import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
+import {
+  SERVICE_STATE_FILE,
+  SERVICE_STOP_MARKER_FILE,
+  serviceStateHasPendingUpdate,
+} from "./serviceProtocol.ts";
 import {
   CLOUD_ENDPOINT_RUNTIME_CONFIG,
   CLOUD_LINKED_USER_ID,
@@ -627,6 +635,33 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
   },
 );
 
+// The launcher records an in-flight update in its durable state file. Read
+// straight from disk: the launcher owns that file, and this runs while the
+// server is tearing down. Unreadable or unparseable state means no pending
+// update — the release then proceeds as before.
+//
+// A pending update alone is not proof a replacement server is coming: an
+// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during
+// the pending window also tears this server down, with nothing starting
+// afterwards. The launcher marks that case with a stop-marker file just
+// before it signals the child, so pending + no marker is the update handoff.
+export const pendingUpdateHandoffExists = Effect.gen(function* () {
+  const config = yield* ServerConfig.ServerConfig;
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const runtimeDir = path.join(config.baseDir, "runtime");
+  const stateText = yield* fs
+    .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
+    .pipe(Effect.option);
+  if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) {
+    return false;
+  }
+  const stopping = yield* fs
+    .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
+    .pipe(Effect.orElseSucceed(() => false));
+  return !stopping;
+});
+
 // Cloudflare bills per provisioned tunnel, so an environment that goes offline
 // must not leave its tunnel behind. Releasing deletes only the tunnel — the
 // relay keeps the link and its hostname reservation, and the next startup's
@@ -649,6 +684,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
   if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
     return false;
   }
+  // A shutdown that hands off to a pending remote update is not the
+  // environment going offline: the launcher immediately brings a server back
+  // (the new version, or the old one after a rollback). Deleting the tunnel
+  // here forces that server to provision a replacement UUID, and the public
+  // hostname's route to the new tunnel takes 1-2 minutes to propagate — the
+  // dominant cost of an update restart. Keep the tunnel instead: the next
+  // boot respawns the connector from the stored config and is reachable as
+  // soon as it connects, and the reconcile confirms the still-live tunnel
+  // without replacing it.
+  if (yield* pendingUpdateHandoffExists) {
+    yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
+    return false;
+  }
   const token = yield* dependencies.cliTokenManager.getExisting;
   if (Option.isNone(token)) {
     return false;

apps/server/src/cloud/serviceProtocol.ts

@@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;
 export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
 export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
 export const SERVICE_STATE_FILE = "service-state.json";
+/** Written by the launcher just before an explicit stop kills its child, so
+    the child can tell "the service is going away" from "the launcher is about
+    to start my replacement" while a pending update is recorded. */
+export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
 
 export interface PendingServiceUpdate {
   readonly id: string;

apps/server/src/server.ts

@@ -83,6 +83,7 @@ import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
 import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
 import {
   connectHttpApiLayer,
+  pendingUpdateHandoffExists,
   reconcileDesiredCloudLink,
   releaseManagedTunnelOnShutdown,
 } from "./cloud/http.ts";
@@ -559,26 +560,34 @@ export const makeServerLayer = Layer.unwrap(
           yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie);
           return;
         }
+        const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe(
+          Effect.timeout("10 seconds"),
+          Effect.tap((released) =>
+            released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void,
+          ),
+          Effect.catchCause((cause) =>
+            Effect.logWarning(
+              "Failed to release the managed tunnel on shutdown; the next link reuses it",
+              { cause },
+            ),
+          ),
+          Effect.asVoid,
+        );
+        // A launcher trial can be stopped before activation. The previous
+        // server is already gone, so the trial owns cleanup immediately; the
+        // pending-state check keeps the tunnel for normal commit or rollback,
+        // while the launcher's explicit-stop marker allows it to be released.
+        // Other runtimes wait for activation so a failed standby cannot tear
+        // down the active runtime's tunnel.
+        const cleanupBeforeActivation = yield* pendingUpdateHandoffExists;
+        if (cleanupBeforeActivation) {
+          yield* Effect.addFinalizer(() => releaseManagedTunnel);
+        }
         yield* forkParked(
           Effect.gen(function* () {
-            // Only an activated runtime owns the tunnel cleanup finalizer.
-            yield* Effect.addFinalizer(() =>
-              releaseManagedTunnelOnShutdown().pipe(
-                Effect.timeout("10 seconds"),
-                Effect.tap((released) =>
-                  released
-                    ? Effect.logInfo("Released the managed tunnel on shutdown")
-                    : Effect.void,
-                ),
-                Effect.catchCause((cause) =>
-                  Effect.logWarning(
-                    "Failed to release the managed tunnel on shutdown; the next link reuses it",
-                    { cause },
-                  ),
-                ),
-                Effect.asVoid,
-              ),
-            );
+            if (!cleanupBeforeActivation) {
+              yield* Effect.addFinalizer(() => releaseManagedTunnel);
+            }
             if (!(yield* CloudCliState.readCliDesiredCloudLink)) return;
             const server = yield* HttpServer.HttpServer;
             const address = server.address;

apps/server/src/serviceLauncher.test.ts

@@ -10,6 +10,7 @@ import {
   decodeServiceState,
   isExactServiceVersion,
   SERVICE_LAUNCHER_PROTOCOL,
+  SERVICE_STOP_MARKER_FILE,
 } from "./cloud/serviceProtocol.ts";
 
 it("accepts only exact semantic versions", () => {
@@ -113,6 +114,9 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
       const running = launcher.run();
       yield* Effect.promise(() => launcher.stop("SIGTERM"));
       yield* Effect.promise(() => running);
+      // An explicit stop leaves the marker that tells a child shutting down
+      // mid-update that no replacement server is coming.
+      assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE)));
     }),
   );
 

apps/server/src/serviceLauncher.ts

@@ -24,6 +24,7 @@ import {
   SERVICE_LAUNCHER_CONTEXT_ENV,
   SERVICE_LAUNCHER_PROTOCOL,
   SERVICE_STATE_FILE,
+  SERVICE_STOP_MARKER_FILE,
 } from "./cloud/serviceProtocol.ts";
 
 const HANDOFF_DELAY_MS = 2_000;
@@ -257,6 +258,9 @@ async function terminateChild(
   }
 }
 
+const stopMarkerPath = (baseDir: string) =>
+  NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
+
 export class Launcher {
   readonly #baseDir: string;
   readonly #statePath: string;
@@ -315,6 +319,15 @@ export class Launcher {
     this.#stopping = true;
     this.#clearTimer();
     this.#enqueue(async () => {
+      // An explicit stop is the service going away, not an update handoff.
+      // The marker lets the child's shutdown release the managed tunnel even
+      // while an update is recorded as pending; the next launcher start
+      // clears it. Written before the child is signalled so the child sees
+      // it during teardown. Best-effort: if the write fails the child errs
+      // toward keeping the tunnel, and the next link or unlink reconciles it.
+      await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch(
+        () => undefined,
+      );
       const child = this.#child?.process;
       this.#child = null;
       if (child !== undefined) await terminateChild(child, signal);
@@ -330,6 +343,10 @@ export class Launcher {
   }
 
   async #recover(): Promise<void> {
+    // A fresh launcher means servers are running again: any stop marker from
+    // a previous explicit stop is stale and must not make a future update
+    // handoff release its tunnel.
+    await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
     const update = this.#state.update;
     if (update?.status !== "pending") {
       if (update !== undefined) {