feat(server): prune sandbox export artifacts and document the sandbox runtime

.env.example

@@ -29,3 +29,24 @@ T3CODE_RELAY_URL=https://relay.t3.codes
 # T3CODE_MOBILE_OTLP_TRACES_URL=https://api.axiom.co/v1/traces
 # T3CODE_MOBILE_OTLP_TRACES_DATASET=t3-code-mobile-traces-dev
 # T3CODE_MOBILE_OTLP_TRACES_TOKEN=xaat-...
+
+# Optional: per-thread container sandboxing (server runtime, all off by default).
+# See docs/operations/sandbox-runtime.md for lifecycle and retention details.
+# Master switch: digest-pinned workspace image; unset = threads run on the host.
+# T3_SANDBOX_IMAGE=registry.example/t3-sandbox@sha256:...
+# Required alongside T3_SANDBOX_IMAGE: preview/signaling sidecar image.
+# T3_SANDBOX_PREVIEW_PROXY_IMAGE=registry.example/t3-preview-proxy@sha256:...
+# Opt-in outbound egress sidecar; unset = sandboxes have no egress.
+# T3_SANDBOX_EGRESS_PROXY_IMAGE=registry.example/t3-egress-proxy@sha256:...
+# Comma-separated sha256 digests of cache volumes the server will mount.
+# T3_SANDBOX_TRUSTED_CACHE_DIGESTS=
+# Override for the export artifact directory; defaults to <stateDir>/sandbox-artifacts.
+# T3_SANDBOX_ARTIFACT_DIR=
+# Ceiling on an archived provider conversation store, in bytes (default 50MB).
+# T3_SANDBOX_STORE_MAX_BYTES=52428800
+# Age cap on exported artifact sets, in seconds (default 30 days; 0 disables the sweep).
+# T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS=2592000
+# "disabled" runs sandboxes headless: no X server, no desktop streaming.
+# T3_SANDBOX_DESKTOP=disabled
+# "disabled" omits container/volume storage quotas on hosts without XFS pquota.
+# T3_SANDBOX_CONTAINER_STORAGE_QUOTA=disabled

apps/server/integration/OrchestrationEngineHarness.integration.ts

@@ -491,6 +491,7 @@ export const makeOrchestrationIntegrationHarness = (
         recoverPreview: () => Effect.die("recoverPreview should not be called in this test"),
         revokeCredentials: () => Effect.succeed(0),
         removeThreadArtifacts: () => Effect.void,
+        sweepExpiredArtifacts: () => Effect.succeed(0),
       });
     }
     const projectFileLoaderLayer = Layer.succeed(T3ProjectFileLoader, {

apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts

@@ -496,6 +496,7 @@ describe("ProviderCommandReactor", () => {
           recoverPreview: () => Effect.succeed(false),
           revokeCredentials: () => Effect.succeed(0),
           removeThreadArtifacts: () => Effect.void,
+          sweepExpiredArtifacts: () => Effect.succeed(0),
         } satisfies SandboxRuntimeManagerShape),
       ),
       // The real directory over the in-memory database: the reactor clears a

apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts

@@ -140,6 +140,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             provision,
             reconcile: () =>
               Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }),
@@ -249,6 +250,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
           ),
           Layer.provide(
             Layer.succeed(SandboxRuntimeManager, {
+              sweepExpiredArtifacts: () => Effect.succeed(0),
               provision: () => Effect.die("runtime must not run without an image"),
               reconcile: () =>
                 Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }),
@@ -340,6 +342,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             exportBranch: () =>
               Effect.succeed({
                 commit: "b".repeat(40),
@@ -441,6 +444,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             stop: () => Effect.die("no container exists for an unprovisioned sandbox"),
             exportBranch: () => Effect.die("nothing to export without a container"),
           } as never),
@@ -540,6 +544,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             exportBranch: () =>
               Effect.succeed({
                 commit: "b".repeat(40),
@@ -643,6 +648,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             provision: () => Effect.die("provisioning is not reached on this path"),
             reconcile: () =>
               Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }),
@@ -742,6 +748,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
         ),
         Layer.provide(
           Layer.succeed(SandboxRuntimeManager, {
+            sweepExpiredArtifacts: () => Effect.succeed(0),
             stop: (_runtime: string, id: string) =>
               Effect.sync(() => {
                 stopped.push(id);
@@ -787,4 +794,107 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => {
       expect(dispatched.map((command) => command.type)).toEqual(["sandbox.stop.complete"]);
     }),
   );
+
+  it.effect(
+    "the periodic pass sweeps expired artifacts, shields active threads, and survives a sweep failure",
+    () =>
+      Effect.gen(function* () {
+        // The sweep rides the same periodic pass as expiry. Threads whose
+        // sandbox is in a non-terminal lifecycle must arrive in the protected
+        // set (their artifact set may seed a re-provision), terminal ones must
+        // not -- and a failing sweep must not stall the rest of the pass.
+        const swept = yield* Deferred.make<ReadonlySet<string>>();
+        const continued = yield* Deferred.make<void>();
+        const dispatched: OrchestrationCommand[] = [];
+        const OLD = "2026-08-15T00:00:00.000Z";
+        const sandboxOf = (lifecycle: "ready" | "stopped") => ({
+          lifecycle,
+          runtime: "podman" as const,
+          branch: { branchName: "t3/thread/x", baseCommit: "a".repeat(40) },
+          limits: {
+            cpuCount: 2,
+            memoryBytes: 4_294_967_296,
+            diskBytes: 21_474_836_480,
+            processCount: 512,
+            idleTimeoutSeconds: 60,
+            maximumLifetimeSeconds: 60,
+          },
+          desktop: { status: "unavailable" as const },
+          services: [],
+          controller: { kind: "none" as const },
+          createdAt: OLD,
+          lastActiveAt: OLD,
+        });
+        const activeThread = { ...snapshot.threads[0]!, sandbox: sandboxOf("ready") };
+        const stoppedThread = {
+          ...snapshot.threads[0]!,
+          id: ThreadId.make("thread-stopped"),
+          sandbox: sandboxOf("stopped"),
+        };
+        const layer = Layer.effect(SandboxLifecycleReactor, make).pipe(
+          Layer.provide(NodeServices.layer),
+          Layer.provide(
+            Layer.mock(GitWorkflowService)({
+              localStatus: () => Effect.succeed({ isRepo: true, refName: "main" } as never),
+            }),
+          ),
+          Layer.provide(Layer.mock(ProviderService)({ listSessions: () => Effect.succeed([]) })),
+          Layer.provide(
+            Layer.succeed(T3ProjectFileLoader, { load: () => Effect.succeed(Option.none()) }),
+          ),
+          Layer.provide(
+            Layer.succeed(SandboxRuntimeManager, {
+              sweepExpiredArtifacts: (protectedThreadIds: ReadonlySet<string>) =>
+                Deferred.succeed(swept, protectedThreadIds).pipe(
+                  Effect.andThen(
+                    Effect.fail(new SandboxManagerError({ message: "artifact directory io" })),
+                  ),
+                ),
+              // Called for every ready/paused thread right after the sweep --
+              // reaching it proves the sweep failure was contained.
+              sampleUsage: () =>
+                Deferred.succeed(continued, undefined).pipe(
+                  Effect.andThen(Effect.fail(new SandboxManagerError({ message: "no container" }))),
+                ),
+              reconcile: () =>
+                Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }),
+            } as never),
+          ),
+          Layer.provide(
+            Layer.mock(ProjectionSnapshotQuery)({
+              getSnapshot: () =>
+                Effect.succeed({
+                  ...snapshot,
+                  threads: [activeThread, stoppedThread],
+                } as typeof snapshot),
+              getThreadDetailById: () => Effect.succeed(Option.none()),
+            }),
+          ),
+          Layer.provide(
+            Layer.mock(OrchestrationEngineService)({
+              dispatch: (command) =>
+                Effect.gen(function* () {
+                  dispatched.push(command);
+                  return { sequence: dispatched.length };
+                }),
+              streamDomainEvents: Stream.empty,
+            }),
+          ),
+        );
+
+        yield* Effect.scoped(
+          Effect.gen(function* () {
+            const reactor = yield* SandboxLifecycleReactor;
+            yield* reactor.start();
+            const protectedIds = yield* Deferred.await(swept).pipe(Effect.timeout("5 seconds"));
+            // The ready thread is shielded; the stopped one is fair game.
+            expect([...protectedIds]).toEqual([threadId]);
+            // The pass reached per-thread work after the sweep failed: the
+            // failure was logged and contained, not propagated.
+            yield* Deferred.await(continued).pipe(Effect.timeout("5 seconds"));
+            yield* reactor.drain;
+          }).pipe(Effect.provide(layer)),
+        );
+      }),
+  );
 });

apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts

@@ -712,6 +712,30 @@ export const make = Effect.gen(function* () {
 
   const expire = Effect.fn("SandboxLifecycleReactor.expire")(function* () {
     const snapshot = yield* snapshots.getSnapshot();
+    // Piggybacks on the periodic pass rather than owning a timer: exported
+    // artifact sets for threads that settled long ago (or were deleted
+    // out-of-band) otherwise accumulate forever. Threads whose sandbox is
+    // still in a non-terminal lifecycle are protected regardless of age --
+    // their next stop overwrites the set, and deleting it early would cost a
+    // re-provision its restore seed. Best-effort: a sweep failure must not
+    // stall expiry.
+    yield* Effect.suspend(() =>
+      runtimes.sweepExpiredArtifacts(
+        new Set(
+          snapshot.threads
+            .filter(
+              (thread) =>
+                thread.sandbox != null &&
+                !["stopped", "expired", "deleted"].includes(thread.sandbox.lifecycle),
+            )
+            .map((thread) => thread.id),
+        ),
+      ),
+    ).pipe(
+      Effect.catchCause((cause) =>
+        Effect.logWarning("sandbox artifact sweep failed", { cause: Cause.pretty(cause) }),
+      ),
+    );
     const now = DateTime.toEpochMillis(yield* DateTime.now);
     const activeSessions = new Set(
       (yield* providers.listSessions()).map((session) => session.threadId),

apps/server/src/sandbox/SandboxProviderStore.test.ts

@@ -101,6 +101,7 @@ const MUTATED_ENV = [
   "T3_SANDBOX_PREVIEW_PROXY_IMAGE",
   "T3_SANDBOX_CREDENTIAL_PROXY_IMAGE",
   "T3_SANDBOX_STORE_MAX_BYTES",
+  "T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS",
 ] as const;
 const originalEnv = new Map(MUTATED_ENV.map((key) => [key, process.env[key]] as const));
 
@@ -149,10 +150,14 @@ describe("provider conversation store artifacts", () => {
         NodeFS.readFileSync(NodePath.join(root, `${ARTIFACT_ID}.json`), "utf8"),
       ) as {
         store?: string;
+        storeServed?: boolean;
         storeSha256?: string;
         storeBytes?: number;
       };
       expect(manifest.store).toBe(`${ARTIFACT_ID}.store.tar`);
+      // The artifact HTTP route serves only `bundle` and `manifest`; the
+      // manifest must say so rather than advertise a 404.
+      expect(manifest.storeServed).toBe(false);
       expect(manifest.storeSha256).toBe(exported.storeSha256);
       expect(manifest.storeBytes).toBe(STORE_CONTENTS.length);
     }),
@@ -289,3 +294,111 @@ describe("provider conversation store artifacts", () => {
     }),
   );
 });
+
+describe("expired artifact retention sweep", () => {
+  const artifactSet = (root: string, threadId: string, ageMs: number) => {
+    const name = NodeCrypto.createHash("sha256").update(threadId).digest("hex");
+    // @effect-diagnostics-next-line globalDate:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time.
+    const mtime = new Date(Date.now() - ageMs);
+    for (const file of [`${name}.bundle`, `${name}.json`, `${name}.store.tar`]) {
+      const path = NodePath.join(root, file);
+      NodeFS.writeFileSync(path, "artifact", "utf8");
+      NodeFS.utimesSync(path, mtime, mtime);
+    }
+    return name;
+  };
+  const DAY_MS = 24 * 60 * 60 * 1000;
+
+  it.effect("deletes sets past the age cap and keeps young ones", () =>
+    Effect.gen(function* () {
+      const root = makeRoot();
+      const old = artifactSet(root, "thread-old", 45 * DAY_MS);
+      const young = artifactSet(root, "thread-young", 5 * DAY_MS);
+      const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor());
+
+      const removed = yield* manager.sweepExpiredArtifacts(new Set());
+
+      expect(removed).toBe(1);
+      const remaining = NodeFS.readdirSync(root).sort();
+      expect(remaining).toEqual([`${young}.bundle`, `${young}.json`, `${young}.store.tar`]);
+      expect(remaining.some((file) => file.startsWith(old))).toBe(false);
+    }),
+  );
+
+  it.effect("a set is as young as its newest file", () =>
+    Effect.gen(function* () {
+      // Exports rename bundle, store, and manifest together; a set whose
+      // manifest is fresh exported recently even if an older sibling survived
+      // a partial overwrite. It must not be deleted piecemeal.
+      const root = makeRoot();
+      const name = artifactSet(root, "thread-mixed", 45 * DAY_MS);
+      // @effect-diagnostics-next-line globalDateInEffect:off - freshens a real filesystem mtime, which the sweep reads with wall-clock time.
+      const now = new Date();
+      NodeFS.utimesSync(NodePath.join(root, `${name}.json`), now, now);
+      const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor());
+
+      expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0);
+      expect(NodeFS.readdirSync(root)).toHaveLength(3);
+    }),
+  );
+
+  it.effect("keeps a set belonging to an active thread regardless of age", () =>
+    Effect.gen(function* () {
+      // A non-terminal sandbox will overwrite its set on the next stop; in the
+      // meantime that set may be the seed a re-provision restores from.
+      const root = makeRoot();
+      const kept = artifactSet(root, "thread-active", 90 * DAY_MS);
+      artifactSet(root, "thread-gone", 90 * DAY_MS);
+      const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor());
+
+      const removed = yield* manager.sweepExpiredArtifacts(new Set(["thread-active"]));
+
+      expect(removed).toBe(1);
+      expect(NodeFS.readdirSync(root).sort()).toEqual([
+        `${kept}.bundle`,
+        `${kept}.json`,
+        `${kept}.store.tar`,
+      ]);
+    }),
+  );
+
+  it.effect("an explicit zero disables the sweep entirely", () =>
+    Effect.gen(function* () {
+      process.env.T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS = "0";
+      const root = makeRoot();
+      artifactSet(root, "thread-ancient", 400 * DAY_MS);
+      const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor());
+
+      expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0);
+      expect(NodeFS.readdirSync(root)).toHaveLength(3);
+    }),
+  );
+
+  it.effect("ignores in-flight export temporaries and foreign files", () =>
+    Effect.gen(function* () {
+      const root = makeRoot();
+      // @effect-diagnostics-next-line globalDateInEffect:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time.
+      const past = new Date(Date.now() - 60 * DAY_MS);
+      for (const file of [`.${"a".repeat(64)}.1234.bundle.tmp`, "seeds"]) {
+        const path = NodePath.join(root, file);
+        NodeFS.writeFileSync(path, "not-an-artifact", "utf8");
+        NodeFS.utimesSync(path, past, past);
+      }
+      const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor());
+
+      expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0);
+      expect(NodeFS.readdirSync(root)).toHaveLength(2);
+    }),
+  );
+
+  it.effect("sweeps nothing without configured artifact storage or a directory", () =>
+    Effect.gen(function* () {
+      const unconfigured = makeSandboxRuntimeManager(undefined, "linux", new FakeExecutor());
+      expect(yield* unconfigured.sweepExpiredArtifacts(new Set())).toBe(0);
+      // A root that exists in config but was never written to (no exports yet).
+      const missing = NodePath.join(makeRoot(), "never-created");
+      const manager = makeSandboxRuntimeManager(missing, "linux", new FakeExecutor());
+      expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0);
+    }),
+  );
+});