feat(server): t3 doctor reclaims database space

apps/server/src/binCli.ts

@@ -1,3 +1,4 @@
+import { doctorCommand } from "./cli/doctor.ts";
 import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as Effect from "effect/Effect";
@@ -85,6 +86,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) =>
       servicePreflightCommand,
       themeCommand,
       traceCommand,
+      doctorCommand,
       triageCommand,
       cloudEnabled ? connectCommand : connectUnavailableCommand,
     ]),

apps/server/src/cli/doctor.test.ts

@@ -0,0 +1,52 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { assert, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+
+import * as BootService from "../cloud/bootService.ts";
+import { runDoctor } from "./doctor.ts";
+import { DatabaseMaintenance } from "../persistence/DatabaseMaintenance.ts";
+
+it.effect("--fix reclaims space with the service paused, and --dry-run changes nothing", () =>
+  Effect.gen(function* () {
+    const calls: string[] = [];
+    const doctor = (dryRun: boolean) =>
+      runDoctor("/test/statev2.sqlite", { fix: true, dryRun, interactive: false }).pipe(
+        Effect.provide(
+          Layer.mergeAll(
+            NodeServices.layer,
+            Layer.mock(BootService.BootService)({
+              withStopped: (operation) =>
+                Effect.sync(() => calls.push("pause")).pipe(
+                  Effect.andThen(operation),
+                  Effect.ensuring(Effect.sync(() => calls.push("resume"))),
+                ),
+            }),
+            Layer.succeed(DatabaseMaintenance, {
+              // An unconverted database: reclaiming space is recommended.
+              inspect: () =>
+                Effect.succeed({
+                  databaseBytes: 2000,
+                  reclaimableBytes: 1000,
+                  incrementalVacuumEnabled: false,
+                }),
+              compact: () =>
+                Effect.sync(() => {
+                  calls.push("compact");
+                  return {
+                    beforeBytes: 2000,
+                    afterBytes: 1000,
+                    deletedEventCount: 1,
+                    deletedReceiptCount: 0,
+                  };
+                }),
+            }),
+          ),
+        ),
+      );
+    yield* doctor(true);
+    assert.deepEqual(calls, []);
+    yield* doctor(false);
+    assert.deepEqual(calls, ["pause", "compact", "resume"]);
+  }),
+);

apps/server/src/cli/doctor.ts

@@ -0,0 +1,121 @@
+import { DATABASE_COMPACTION_RECOMMENDED_BYTES } from "@t3tools/contracts";
+import * as Console from "effect/Console";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { Command, Flag, GlobalFlag, Prompt } from "effect/cli";
+
+import * as BootService from "../cloud/bootService.ts";
+import * as DatabaseMaintenance from "../persistence/DatabaseMaintenance.ts";
+import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";
+import * as CliService from "./service.ts";
+
+const formatBytes = (bytes: number) =>
+  bytes >= 1024 ** 3
+    ? `${(bytes / 1024 ** 3).toFixed(2)} GiB`
+    : bytes >= 1024 ** 2
+      ? `${(bytes / 1024 ** 2).toFixed(1)} MiB`
+      : `${Math.ceil(bytes / 1024)} KiB`;
+
+export const runDoctor = Effect.fn("runDoctor")(function* (
+  databasePath: string,
+  options: { readonly fix: boolean; readonly dryRun: boolean; readonly interactive?: boolean },
+) {
+  const maintenance = yield* DatabaseMaintenance.DatabaseMaintenance;
+  const status = yield* maintenance.inspect(databasePath);
+  yield* Console.log(
+    `Database: ${formatBytes(status.databaseBytes)}, ${formatBytes(status.reclaimableBytes)} reclaimable.`,
+  );
+  if (status.incrementalVacuumEnabled && status.reclaimableBytes === 0) {
+    yield* Console.log("Nothing to repair.");
+    return;
+  }
+  const recommended =
+    !status.incrementalVacuumEnabled ||
+    status.reclaimableBytes >= DATABASE_COMPACTION_RECOMMENDED_BYTES;
+  const repairs = [
+    {
+      id: "database" as const,
+      title: "Reclaim database space",
+      description: status.incrementalVacuumEnabled
+        ? "Returns unused space to disk."
+        : "Returns unused space to disk and turns on automatic shrinking.",
+      recommended,
+    },
+  ];
+  for (const repair of repairs) {
+    yield* Console.log(
+      `\n${repair.title}${repair.recommended ? " (recommended)" : ""}\n  ${repair.description}\n  Pauses the background service, if installed, while it runs and may need temporary disk space.`,
+    );
+  }
+  if (options.dryRun) {
+    yield* Console.log("\nDry run: nothing changed.");
+    return;
+  }
+  const interactive = options.interactive ?? Boolean(process.stdin.isTTY && process.stdout.isTTY);
+  if (!options.fix && !interactive) {
+    yield* Console.log(
+      "\nRun in a terminal to choose repairs, or pass --fix for the recommended ones.",
+    );
+    return;
+  }
+  const selected = options.fix
+    ? repairs.filter((repair) => repair.recommended).map((repair) => repair.id)
+    : yield* Prompt.run(
+        Prompt.MultiSelect({
+          message: "Choose repairs to run",
+          choices: repairs.map((repair) => ({
+            title: repair.title + (repair.recommended ? " (recommended)" : ""),
+            value: repair.id,
+            selected: repair.recommended,
+          })),
+        }),
+      ).pipe(Effect.catchTags({ QuitError: () => Effect.succeed([]) }));
+  if (selected.length === 0) {
+    yield* Console.log(
+      options.fix
+        ? "No recommended repairs; nothing changed."
+        : "No repairs selected; nothing changed.",
+    );
+    return;
+  }
+  const service = yield* BootService.BootService;
+  yield* Console.log("\nReclaiming database space. This can take several minutes...");
+  // Report inside the pause: a repair that succeeded is worth knowing about
+  // even when starting the service again fails afterwards.
+  yield* service.withStopped(
+    maintenance
+      .compact(databasePath)
+      .pipe(
+        Effect.tap((result) =>
+          Console.log(
+            `Done: ${formatBytes(result.beforeBytes)} -> ${formatBytes(result.afterBytes)}, ${result.deletedEventCount} superseded events removed.`,
+          ),
+        ),
+      ),
+  );
+});
+
+export const doctorCommand = Command.make("doctor", {
+  ...projectLocationFlags,
+  fix: Flag.Boolean("fix").pipe(
+    Flag.withDescription("Apply the recommended repairs without prompting."),
+    Flag.withDefault(false),
+  ),
+  dryRun: Flag.Boolean("dry-run").pipe(
+    Flag.withDescription("Show the repairs without changing anything, even with --fix."),
+    Flag.withDefault(false),
+  ),
+}).pipe(
+  Command.withDescription(
+    "Inspect this T3 Code data directory and repair it. Pauses and restarts the background service as needed.",
+  ),
+  Command.withHandler((flags) =>
+    Effect.gen(function* () {
+      const logLevel = yield* GlobalFlag.LogLevel;
+      const config = yield* resolveCliAuthConfig(flags, logLevel);
+      yield* runDoctor(config.dbPath, { fix: flags.fix, dryRun: flags.dryRun }).pipe(
+        Effect.provide(Layer.merge(DatabaseMaintenance.layer, CliService.layer(config))),
+      );
+    }),
+  ),
+);

apps/server/src/cli/service.test.ts

@@ -102,6 +102,7 @@ function makeTestService(serviceStatus: BootService.BootServiceStatus) {
   const restarts: Array<true> = [];
   const service = BootService.BootService.of({
     status: Effect.succeed(serviceStatus),
+    withStopped: (operation) => operation,
     restart: Effect.sync(() => {
       restarts.push(true);
       return serviceStatus.installed;

apps/server/src/cloud/bootService.test.ts

@@ -168,6 +168,8 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
         control.linger = "yes";
       if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true;
       if (!failed && command === "systemctl --user restart t3code.service") control.active = true;
+      // A stop that fails (a timeout, say) has still stopped the service.
+      if (command === "systemctl --user stop t3code.service") control.active = false;
       if (
         control.stateAfterStop !== undefined &&
         (command === "systemctl --user stop t3code.service" ||
@@ -190,7 +192,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
                 : "",
         stderr: "",
         code: ChildProcessSpawner.ExitCode(
-          failed || (input.args[1] === "is-active" && !control.active) ? 1 : 0,
+          failed ||
+            (input.args[1] === "is-active" && !control.active) ||
+            (input.args[1] === "is-enabled" && !control.enabled)
+            ? 1
+            : 0,
         ),
         timedOut: false,
         stdoutTruncated: false,
@@ -560,6 +566,73 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
     }),
   );
 
+  it.effect("pauses only this home's running service for maintenance, even when it fails", () =>
+    Effect.gen(function* () {
+      const { service, commands, control, makeService } = yield* makeHarness();
+      yield* service.install();
+      const failure = yield* service
+        .withStopped(
+          Effect.suspend(() => Effect.fail(control.active ? "running" : "repair failed")),
+        )
+        .pipe(Effect.flip);
+      expect(failure).toBe("repair failed");
+      expect(control.active).toBe(true);
+
+      // A failed stop skips the operation but still starts the service again.
+      control.failCommand = "systemctl --user stop t3code.service";
+      let ran = false;
+      const stopFailure = yield* service
+        .withStopped(
+          Effect.sync(() => {
+            ran = true;
+          }),
+        )
+        .pipe(Effect.flip);
+      expect(stopFailure).toMatchObject({
+        _tag: "BootServiceCommandError",
+        step: "stopping the installed service",
+      });
+      expect(ran).toBe(false);
+      expect(control.active).toBe(true);
+      control.failCommand = undefined;
+
+      // An enabled unit that is not active (crash-looping in auto-restart, say)
+      // is still stopped for the repair and started again afterwards.
+      control.active = false;
+      commands.length = 0;
+      yield* service.withStopped(Effect.sync(() => commands.push("repair")));
+      expect(commands).toEqual([
+        "systemctl --user is-enabled t3code.service",
+        "systemctl --user is-active t3code.service",
+        "systemctl --user stop t3code.service",
+        "repair",
+        "systemctl --user restart t3code.service",
+      ]);
+      expect(control.active).toBe(true);
+      // A running unit the user disabled is started again but stays disabled.
+      control.enabled = false;
+      yield* service.withStopped(Effect.void);
+      expect(control).toMatchObject({ active: true, enabled: false });
+      // A unit that was neither enabled nor active stays stopped.
+      control.active = false;
+      control.enabled = false;
+      commands.length = 0;
+      yield* service.withStopped(Effect.sync(() => commands.push("repair")));
+      expect(commands).toContain("systemctl --user stop t3code.service");
+      expect(commands).not.toContain("systemctl --user restart t3code.service");
+      expect(control.active).toBe(false);
+      control.enabled = true;
+      control.active = true;
+
+      const fs = yield* FileSystem.FileSystem;
+      const otherHome = yield* fs.makeTempDirectoryScoped({ prefix: "t3-doctor-other-" });
+      const other = yield* makeService(undefined, "1.2.3", otherHome);
+      commands.length = 0;
+      expect(yield* other.withStopped(Effect.sync(() => control.active))).toBe(true);
+      expect(commands).toEqual([]);
+    }),
+  );
+
   it.effect("restart stops and starts an installed service, and is a no-op otherwise", () =>
     Effect.gen(function* () {
       const { service, commands } = yield* makeHarness();

apps/server/src/cloud/bootService.ts

@@ -4,6 +4,7 @@ import * as Context from "effect/Context";
 import * as DateTime from "effect/DateTime";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
@@ -547,6 +548,16 @@ export class BootService extends Context.Service<
      * restarted.
      */
     readonly restart: Effect.Effect<boolean, BootServiceError>;
+    /**
+     * Run `operation` while this home's installed service is stopped, then
+     * start it again when it was enabled (systemd) or loaded (launchd)
+     * beforehand. A service that belongs to another home is left alone. The
+     * service is started again even when stopping it failed; the operation's
+     * own failure is reported ahead of a failure to restart.
+     */
+    readonly withStopped: <A, E, R>(
+      operation: Effect.Effect<A, E, R>,
+    ) => Effect.Effect<A, E | BootServiceError, R>;
     readonly uninstall: Effect.Effect<boolean, BootServiceError>;
     readonly status: Effect.Effect<BootServiceStatus, BootServiceError>;
   }
@@ -931,6 +942,61 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
     Effect.withSpan("cloud.boot_service.restart"),
   );
 
+  const withStopped = <A, E, R>(
+    operation: Effect.Effect<A, E, R>,
+  ): Effect.Effect<A, E | BootServiceError, R> =>
+    Effect.gen(function* () {
+      if (detectedManager === undefined) return yield* operation;
+      const manager = detectedManager;
+      const unit = yield* fs.readFileString(unitPath).pipe(Effect.option);
+      if (Option.isNone(unit)) return yield* operation;
+      const installedBaseDir = bootServiceBaseDirOf(unit.value);
+      if (
+        installedBaseDir === undefined ||
+        path.resolve(installedBaseDir) !== path.resolve(input.baseDir)
+      ) {
+        return yield* operation;
+      }
+      // `is-active` fails while a crash-looping unit sits in `activating
+      // (auto-restart)`, and a throttled launchd job is loaded without a pid;
+      // both would start the server mid-repair. So the unit is always
+      // stopped, and started again when it was active, enabled or loaded
+      // before; a disabled unit caught crash-looping stays stopped.
+      const restore =
+        manager.kind === "systemd"
+          ? yield* Effect.map(
+              Effect.all(
+                [
+                  probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]),
+                  probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]),
+                ],
+                { concurrency: "unbounded" },
+              ),
+              ([enabled, active]) => succeeded(active) || succeeded(enabled),
+            )
+          : succeeded(
+              yield* probe("launchctl", ["print", `gui/${uid}/${BOOT_SERVICE_LAUNCHD_LABEL}`]),
+            );
+      return yield* Effect.acquireUseRelease(
+        // Every stop step is required when the service is to be restored: a
+        // step that was fine to skip during uninstall would leave the server
+        // holding the database. A stop that fails (a timeout, say) may still
+        // have stopped the service, so the restore below runs either way.
+        // Nothing to restore means nothing is expected to be loaded, and the
+        // stop is best-effort (launchd's bootout fails on an unloaded job).
+        runSteps(manager.stop.map((step) => ({ ...step, optional: !restore }))).pipe(Effect.exit),
+        (stopped): Effect.Effect<A, E | BootServiceError, R> =>
+          Exit.isSuccess(stopped) ? operation : Effect.failCause(stopped.cause),
+        (_, exit) => {
+          if (!restore) return Effect.void;
+          // Start only: `activate` would also enable a unit the user disabled.
+          // When the operation itself failed, its error is the one worth reporting.
+          const start = runSteps(manager.restart);
+          return Exit.isFailure(exit) ? Effect.ignore(start) : start;
+        },
+      );
+    }).pipe(Effect.withSpan("cloud.boot_service.with_stopped"));
+
   const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () {
     const manager = yield* requireManager;
     if (
@@ -994,7 +1060,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
     Effect.withSpan("cloud.boot_service.status"),
   );
 
-  return BootService.of({ install, restart, uninstall, status });
+  return BootService.of({ install, restart, withStopped, uninstall, status });
 });
 
 export const layer = (input: {