fix(server): t3 update and t3 service restart run from inside T3 no longer leave the service stopped

apps/server/src/cloud/bootService.test.ts

@@ -171,7 +171,12 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
       if (!failed && command === "loginctl enable-linger --no-ask-password 501")
         control.linger = "yes";
       if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true;
-      if (!failed && command === "systemctl --user restart t3code.service") control.active = true;
+      if (
+        !failed &&
+        (command === "systemctl --user restart t3code.service" ||
+          command === "systemctl --user restart --no-block t3code.service")
+      )
+        control.active = true;
       if (
         control.stateAfterStop !== undefined &&
         (command === "systemctl --user stop t3code.service" ||
@@ -208,6 +213,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
     environmentPath: string | undefined = installerPath,
     cliVersion = "1.2.3",
     serviceBaseDir = baseDir,
+    insideService = false,
   ) =>
     Effect.gen(function* () {
       // Every version the tests install is present and verified on disk, so
@@ -240,6 +246,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
                 ...(environmentPath === undefined || environmentPath === ""
                   ? {}
                   : { PATH: environmentPath }),
+                ...(insideService ? { T3_BOOT_SERVICE_UNIT: "t3code.service" } : {}),
               },
             }),
           ),
@@ -622,6 +629,73 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
     }),
   );
 
+  it.effect("hands an install from inside the service to a transient unit", () =>
+    Effect.gen(function* () {
+      const { service, fs, statePath, commands, makeService } = yield* makeHarness();
+      const path = yield* Path.Path;
+      const plan = yield* service.install();
+      const unit = yield* fs.readFileString(plan.unitPath);
+      const state = yield* fs.readFileString(statePath);
+      commands.length = 0;
+
+      const baseDir = path.dirname(path.dirname(statePath));
+      const newer = yield* makeService(undefined, "1.2.4", undefined, true);
+      yield* newer.install({ allowDowngrade: true });
+
+      expect(
+        commands.filter(
+          (command) =>
+            (command.startsWith("systemctl ") || command.startsWith("systemd-run ")) &&
+            !command.includes("show-environment"),
+        ),
+      ).toEqual([
+        `systemd-run --user --collect --quiet --setenv=T3_BOOT_SERVICE_UNIT= ${pinnedRuntimePaths(path, baseDir, "1.2.4", "linux").entryPath} service install --base-dir ${baseDir} --allow-downgrade`,
+      ]);
+      // The transient unit writes them after stopping the launcher.
+      expect(yield* fs.readFileString(statePath)).toBe(state);
+      expect(yield* fs.readFileString(plan.unitPath)).toBe(unit);
+    }),
+  );
+
+  it.effect("restarts from inside the service by queuing the restart", () =>
+    Effect.gen(function* () {
+      const { service, commands, makeService } = yield* makeHarness();
+      yield* service.install();
+      commands.length = 0;
+
+      const inside = yield* makeService(undefined, "1.2.3", undefined, true);
+      expect(yield* inside.restart).toBe(true);
+      expect(
+        commands.filter(
+          (command) => command.startsWith("systemctl ") && !command.includes("show-environment"),
+        ),
+      ).toEqual([
+        "systemctl --user daemon-reload",
+        "systemctl --user enable t3code.service",
+        "systemctl --user restart --no-block t3code.service",
+      ]);
+    }),
+  );
+
+  it.effect("leaves the service running when a restart handoff from inside it fails", () =>
+    Effect.gen(function* () {
+      const { service, commands, control, makeService } = yield* makeHarness();
+      yield* service.install();
+      commands.length = 0;
+      control.failCommand = "systemctl --user daemon-reload";
+
+      const inside = yield* makeService(undefined, "1.2.3", undefined, true);
+      const error = yield* inside.restart.pipe(Effect.flip);
+      expect(error._tag).toBe("BootServiceCommandError");
+      // Nothing was stopped, so there is nothing to bring back.
+      expect(
+        commands.filter(
+          (command) => command.startsWith("systemctl ") && !command.includes("show-environment"),
+        ),
+      ).toEqual(["systemctl --user daemon-reload"]);
+    }),
+  );
+
   it.effect("restarts an installed service when repair fails", () =>
     Effect.gen(function* () {
       const { service, commands, control } = yield* makeHarness();

apps/server/src/cloud/bootService.ts

@@ -228,6 +228,12 @@ export interface BootServiceManager {
   readonly activate: ReadonlyArray<BootServiceStep>;
   /** Best-effort recovery after a failed repair of an installed service. */
   readonly restart: ReadonlyArray<BootServiceStep>;
+  /**
+   * Replaces stop + activate for `restart` when the caller runs inside the
+   * service, where a stop would kill it before the start. The last entry
+   * queues the restart with the service manager instead of waiting for it.
+   */
+  readonly handoff?: ReadonlyArray<BootServiceStep>;
   /** Uninstall, before the unit file is removed. */
   readonly deactivate: ReadonlyArray<BootServiceStep>;
   /** Uninstall, after the unit file is removed. */
@@ -282,6 +288,25 @@ function systemdManager(input: {
         args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE],
       },
     ],
+    handoff: [
+      {
+        step: "reloading systemd user units",
+        command: "systemctl",
+        args: ["--user", "daemon-reload"],
+      },
+      {
+        step: "enabling the service",
+        command: "systemctl",
+        args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE],
+      },
+      // The user manager owns the queued job and finishes it after the stop
+      // has killed this process along with the rest of the unit.
+      {
+        step: "restarting the service",
+        command: "systemctl",
+        args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE],
+      },
+    ],
     deactivate: [
       {
         step: "stopping the service",
@@ -568,6 +593,11 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
   );
   const homeDir = yield* Config.String("HOME").pipe(Config.withDefault(""));
   const installerPath = yield* Config.String("PATH").pipe(Config.withDefault(""));
+  // The unit exports its name to everything it runs, so this is set for agent
+  // tool calls and terminals started by the service.
+  const runningInsideService =
+    (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) ===
+    BOOT_SERVICE_UNIT_FILE;
   const fs = yield* FileSystem.FileSystem;
   const path = yield* Path.Path;
   const runner = yield* ProcessRunner.ProcessRunner;
@@ -824,6 +854,24 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
     // written, from the same read the downgrade check uses; the stop that
     // normally serialises against the launcher is skipped on purpose.
     const start = options?.start !== false;
+    if (installed && start && runningInsideService && manager.kind === "systemd") {
+      // Stopping the unit from inside it kills this process before the start.
+      // A transient unit runs the same install from outside, stop first, so it
+      // still serialises with a remote update the launcher may be starting.
+      yield* runStep("handing the install to a transient unit", "systemd-run", [
+        "--user",
+        "--collect",
+        "--quiet",
+        `--setenv=${BOOT_SERVICE_UNIT_ENV}=`,
+        runtimePaths.entryPath,
+        "service",
+        "install",
+        "--base-dir",
+        input.baseDir,
+        ...(options?.allowDowngrade === true ? ["--allow-downgrade"] : []),
+      ]);
+      return plan;
+    }
     if (installed && start) {
       yield* runSteps(manager.stop);
     }
@@ -912,6 +960,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
     ) {
       return false;
     }
+    // Writes no state, and the launcher lets an update transition in flight
+    // finish before the queued stop tears it down.
+    if (runningInsideService && manager.handoff !== undefined) {
+      yield* runSteps(manager.handoff);
+      return true;
+    }
     yield* runSteps(manager.stop);
     yield* runSteps(manager.activate).pipe(
       // Same recovery as a failed repair: a service that was running should