feat(server): t3 service protect keeps the server alive when agents fill memory

apps/server/src/cli/service.test.ts

@@ -90,6 +90,37 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands
       }),
   );
 
+  it.effect("protect refuses root before creating any T3 directories", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" });
+      const baseDir = `${parent}/t3`;
+      vi.spyOn(BootService, "layer").mockReturnValue(
+        Layer.succeed(BootService.BootService, makeTestService(status).service),
+      );
+
+      const output = yield* Command.runWith(serviceCommand, { version: packageJson.version })([
+        "protect",
+        "--base-dir",
+        baseDir,
+      ]).pipe(
+        Effect.andThen(TestConsole.logLines),
+        Effect.map((lines) => lines.join("\n")),
+        Effect.provideService(HostProcess.Environment, {}),
+        Effect.provideService(HostProcess.Platform, "linux"),
+        Effect.provideService(HostProcess.UserId, 0),
+        Effect.provide(
+          Layer.mergeAll(
+            ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })),
+            Layer.fresh(TestConsole.layer),
+          ),
+        ),
+      );
+      expect(output).toContain("without sudo");
+      expect(yield* fs.exists(baseDir)).toBe(false);
+    }),
+  );
+
   // regression: #13239 a stale nvm-installed CLI reported "already using" its own version
   it.effect.each(["install", "update"] as const)(
     "%s names the CLI that found nothing to change",

apps/server/src/cli/service.ts

@@ -2,12 +2,14 @@ import * as HostProcess from "@t3tools/shared/HostProcess";
 import * as Console from "effect/Console";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
+import * as Path from "effect/Path";
 import * as Terminal from "effect/Terminal";
 import { Command, Flag, GlobalFlag, Prompt } from "effect/cli";
 import { FetchHttpClient } from "effect/http";
 
 import packageJson from "../../package.json" with { type: "json" };
 import * as BootService from "../cloud/bootService.ts";
+import * as ServiceProtection from "../cloud/serviceProtection.ts";
 import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts";
 import type * as ServerConfig from "../config.ts";
 import * as ProcessRunner from "../processRunner.ts";
@@ -68,7 +70,23 @@ function formatServiceChange(
   return `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`;
 }
 
-function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: string): string {
+function formatProtection(
+  protection: ServiceProtection.ServiceProtectionState | undefined,
+): ReadonlyArray<string> {
+  if (protection === undefined) return [];
+  return [
+    protection.unprotected === undefined
+      ? `  Memory protection: on${protection.setByProtect ? " (t3 service protect)" : ""}`
+      : `  Memory protection: off, ${protection.unprotected} grants none. Run \`t3 service protect\`.`,
+    ...(protection.oomScoreAdj === undefined ? [] : [`  OOM score: ${protection.oomScoreAdj}`]),
+  ];
+}
+
+function formatServiceStatus(
+  status: BootService.BootServiceStatus,
+  cliVersion: string,
+  protection?: ServiceProtection.ServiceProtectionState,
+): string {
   if (!status.supported) {
     return "T3 Code service\n  Status: unavailable on this machine\n  Supported on: Linux with systemd, macOS with launchd";
   }
@@ -89,6 +107,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
       `  Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`,
       `  Unit: ${status.unitPath}`,
       `  Logs: ${status.logPath}`,
+      ...formatProtection(protection),
       ...problems,
       `  Next: Run \`t3 update ${installedVersion}\` to match it, or pass \`--allow-downgrade\` to \`t3 service install\` explicitly.`,
     ].join("\n");
@@ -98,6 +117,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
     `  Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`,
     `  Unit: ${status.unitPath}`,
     `  Logs: ${status.logPath}`,
+    ...formatProtection(protection),
     ...problems,
     ...(status.current ? [] : ["  Next: Run `t3 service install` to repair it."]),
   ].join("\n");
@@ -210,12 +230,121 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe(
       flags,
       Effect.gen(function* () {
         const service = yield* BootService.BootService;
-        yield* Console.log(formatServiceStatus(yield* service.status, packageJson.version));
+        const path = yield* Path.Path;
+        const status = yield* service.status;
+        const protection =
+          status.installed && (yield* HostProcess.Platform) === "linux"
+            ? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe(
+                Effect.provide(ProcessRunner.layer),
+              )
+            : undefined;
+        yield* Console.log(formatServiceStatus(status, packageJson.version, protection));
       }),
     ),
   ),
 );
 
+const formatDropIns = (dropIns: ReadonlyArray<ServiceProtection.DropIn>) =>
+  dropIns.map(
+    ({ path, contents }) => `  ${path}: ${contents.trimEnd().split("\n").slice(1).join(" ")}`,
+  );
+
+const confirm = (yes: boolean, message: string) =>
+  yes ? Effect.succeed(true) : Prompt.run(Prompt.Confirm({ message, initial: false }));
+
+const serviceProtectCommand = Command.make("protect", {
+  ...projectLocationFlags,
+  undo: Flag.Boolean("undo").pipe(
+    Flag.withDescription("Remove the protection this command set up."),
+    Flag.withDefault(false),
+  ),
+  yes: Flag.Boolean("yes").pipe(
+    Flag.withDescription("Answer yes to every question, including SSH and Tailscale."),
+    Flag.withDefault(false),
+  ),
+}).pipe(
+  Command.withDescription(
+    "Protect the T3 Code server from agents under memory pressure (Linux with systemd). Asks for sudo.",
+  ),
+  Command.withHandler((flags) =>
+    Effect.gen(function* () {
+      const unsupported = "Service protection needs Linux with systemd.";
+      if ((yield* HostProcess.Platform) !== "linux") return yield* Console.log(unsupported);
+      // Before resolving config, which creates T3 directories that root would then own.
+      if ((yield* HostProcess.UserId) === 0) {
+        return yield* Console.log(
+          "Run `t3 service protect` as the user that runs T3 Code, without sudo. It asks for sudo when it needs root.",
+        );
+      }
+      return yield* runServiceCommand(
+        flags,
+        Effect.gen(function* () {
+          const uid = yield* HostProcess.UserId;
+          const remote = yield* ServiceProtection.detectRemoteAccessUnits;
+          const { unitPath } = yield* (yield* BootService.BootService).status;
+          if (uid === undefined || remote === undefined || unitPath === "") {
+            return yield* Console.log(unsupported);
+          }
+          const userUnitDir = (yield* Path.Path).dirname(unitPath);
+
+          if (flags.undo) {
+            const installed = yield* ServiceProtection.installedProtection(
+              ServiceProtection.protectionPlan({
+                uid,
+                userUnitDir,
+                remoteAccessUnits: ServiceProtection.REMOTE_ACCESS_UNITS,
+              }),
+            );
+            if (installed === undefined) {
+              return yield* Console.log("T3 Code service protection is not set up.");
+            }
+            yield* Console.log(
+              [
+                "Removes:",
+                ...[...installed.system, ...installed.user, installed.managerDefaults].map(
+                  ({ path, shared }) =>
+                    `  ${path}${shared === true ? " (kept while another user is protected)" : ""}`,
+                ),
+              ].join("\n"),
+            );
+            if (!(yield* confirm(flags.yes, "Remove T3 Code service protection?"))) return;
+            yield* ServiceProtection.removeProtection(installed);
+            return yield* Console.log(
+              "Removed T3 Code service protection. The server's OOM score returns to normal after `t3 service restart`.",
+            );
+          }
+
+          const names = remote.map((unit) => unit.replace(/\.service$/, "")).join(" and ");
+          const protectRemote =
+            remote.length > 0 &&
+            (yield* confirm(
+              flags.yes,
+              `Also protect ${names} so you can still reach this machine under memory pressure?`,
+            ));
+          const plan = ServiceProtection.protectionPlan({
+            uid,
+            userUnitDir,
+            remoteAccessUnits: protectRemote ? remote : [],
+          });
+          yield* Console.log(
+            [
+              "As root, through sudo:",
+              ...formatDropIns(plan.system),
+              "As you:",
+              ...formatDropIns([plan.managerDefaults, ...plan.user]),
+            ].join("\n"),
+          );
+          if (!(yield* confirm(flags.yes, "Apply these settings?"))) return;
+          yield* ServiceProtection.applyProtection(plan);
+          yield* Console.log(
+            "Protected T3 Code. Memory protection applies now; the server takes its OOM score after `t3 service restart`. Undo with `t3 service protect --undo`.",
+          );
+        }).pipe(Effect.provide(ProcessRunner.layer)),
+      );
+    }),
+  ),
+);
+
 export type ServiceOnboardingOutcome = "ready" | "pending" | "skipped";
 
 export const offerServiceDuringOnboarding = Effect.gen(function* () {
@@ -305,6 +434,7 @@ export const serviceCommand = Command.make("service").pipe(
     serviceRestartCommand,
     serviceUninstallCommand,
     serviceStatusCommand,
+    serviceProtectCommand,
     serviceUpdateCommand,
   ]),
 );

apps/server/src/cloud/bootService.test.ts

@@ -56,7 +56,13 @@ it("protects the server unit from agents under CPU and memory pressure", () => {
     unitPath: "/home/theo/.config/systemd/user/t3code.service",
   });
   const service = unit.slice(unit.indexOf("[Service]"), unit.indexOf("[Install]"));
-  for (const line of ["CPUWeight=1000", "IOWeight=1000", "MemoryLow=1536M", "OOMPolicy=continue"]) {
+  for (const line of [
+    "CPUWeight=1000",
+    "IOWeight=1000",
+    "MemoryLow=1536M",
+    "OOMScoreAdjust=100",
+    "OOMPolicy=continue",
+  ]) {
     expect(service.split("\n")).toContain(line);
   }
 });

apps/server/src/cloud/bootService.ts

@@ -116,6 +116,9 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
     // Reclaim takes the agents' unprotected pages first. The kernel caps this at the
     // protection of user.slice and user@.service, which only root can grant.
     "MemoryLow=1536M",
+    // A user manager's own score, the lowest it may set: below its other services (200) and agents (800).
+    // `t3 service protect` lowers it further through a drop-in.
+    "OOMScoreAdjust=100",
     // systemd-oomd ignores this preference when the watched cgroup is root-owned, as user-1000.slice usually is.
     "ManagedOOMPreference=avoid",
     `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,