From e2c345d283807dc2d732809fd4bf0582645bce4a Mon Sep 17 00:00:00 2001 From: awtprod Date: Fri, 21 Aug 2026 07:50:34 +0000 Subject: [PATCH 1/3] feat(server): prune expired sandbox export artifacts on the periodic pass Nothing ever pruned /sandbox-artifacts: every stop/expire wrote a bundle + manifest + provider store set, and sets for threads deleted out-of-band or settled long ago accumulated forever. Add an age-based sweep (T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS, default 30 days, explicit 0 disables) that rides the sandbox lifecycle reactor's existing minutely pass. Sets whose sha matches a thread in a non-terminal sandbox lifecycle are kept regardless of age -- their export may seed a re-provision. A set is dated by its newest file, per-run deletions are capped at 1000 sets, and sweep failures are logged with cause without stalling expiry. Co-Authored-By: Claude Fable 5 --- .../OrchestrationEngineHarness.integration.ts | 1 + .../Layers/ProviderCommandReactor.test.ts | 1 + .../Layers/SandboxLifecycleReactor.test.ts | 110 +++++++++++++++++ .../Layers/SandboxLifecycleReactor.ts | 24 ++++ .../src/sandbox/SandboxProviderStore.test.ts | 109 +++++++++++++++++ .../src/sandbox/SandboxRuntimeManager.ts | 115 +++++++++++++++++- 6 files changed, 356 insertions(+), 4 deletions(-) diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 2edbc6e27a51..164659b732cc 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -491,6 +491,7 @@ export const makeOrchestrationIntegrationHarness = ( recoverPreview: () => Effect.die("recoverPreview should not be called in this test"), revokeCredentials: () => Effect.succeed(0), removeThreadArtifacts: () => Effect.void, + sweepExpiredArtifacts: () => Effect.succeed(0), }); } const projectFileLoaderLayer = Layer.succeed(T3ProjectFileLoader, { diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 91a6628dabe3..a62d0d3b29f2 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -496,6 +496,7 @@ describe("ProviderCommandReactor", () => { recoverPreview: () => Effect.succeed(false), revokeCredentials: () => Effect.succeed(0), removeThreadArtifacts: () => Effect.void, + sweepExpiredArtifacts: () => Effect.succeed(0), } satisfies SandboxRuntimeManagerShape), ), // The real directory over the in-memory database: the reactor clears a diff --git a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts index b5d8516c9dc7..064ef01c4979 100644 --- a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts +++ b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts @@ -140,6 +140,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision, reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -249,6 +250,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision: () => Effect.die("runtime must not run without an image"), reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -340,6 +342,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), exportBranch: () => Effect.succeed({ commit: "b".repeat(40), @@ -441,6 +444,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), stop: () => Effect.die("no container exists for an unprovisioned sandbox"), exportBranch: () => Effect.die("nothing to export without a container"), } as never), @@ -540,6 +544,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), exportBranch: () => Effect.succeed({ commit: "b".repeat(40), @@ -643,6 +648,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision: () => Effect.die("provisioning is not reached on this path"), reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -742,6 +748,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), stop: (_runtime: string, id: string) => Effect.sync(() => { stopped.push(id); @@ -787,4 +794,107 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { expect(dispatched.map((command) => command.type)).toEqual(["sandbox.stop.complete"]); }), ); + + it.effect( + "the periodic pass sweeps expired artifacts, shields active threads, and survives a sweep failure", + () => + Effect.gen(function* () { + // The sweep rides the same periodic pass as expiry. Threads whose + // sandbox is in a non-terminal lifecycle must arrive in the protected + // set (their artifact set may seed a re-provision), terminal ones must + // not -- and a failing sweep must not stall the rest of the pass. + const swept = yield* Deferred.make>(); + const continued = yield* Deferred.make(); + const dispatched: OrchestrationCommand[] = []; + const OLD = "2026-08-15T00:00:00.000Z"; + const sandboxOf = (lifecycle: "ready" | "stopped") => ({ + lifecycle, + runtime: "podman" as const, + branch: { branchName: "t3/thread/x", baseCommit: "a".repeat(40) }, + limits: { + cpuCount: 2, + memoryBytes: 4_294_967_296, + diskBytes: 21_474_836_480, + processCount: 512, + idleTimeoutSeconds: 60, + maximumLifetimeSeconds: 60, + }, + desktop: { status: "unavailable" as const }, + services: [], + controller: { kind: "none" as const }, + createdAt: OLD, + lastActiveAt: OLD, + }); + const activeThread = { ...snapshot.threads[0]!, sandbox: sandboxOf("ready") }; + const stoppedThread = { + ...snapshot.threads[0]!, + id: ThreadId.make("thread-stopped"), + sandbox: sandboxOf("stopped"), + }; + const layer = Layer.effect(SandboxLifecycleReactor, make).pipe( + Layer.provide(NodeServices.layer), + Layer.provide( + Layer.mock(GitWorkflowService)({ + localStatus: () => Effect.succeed({ isRepo: true, refName: "main" } as never), + }), + ), + Layer.provide(Layer.mock(ProviderService)({ listSessions: () => Effect.succeed([]) })), + Layer.provide( + Layer.succeed(T3ProjectFileLoader, { load: () => Effect.succeed(Option.none()) }), + ), + Layer.provide( + Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: (protectedThreadIds: ReadonlySet) => + Deferred.succeed(swept, protectedThreadIds).pipe( + Effect.andThen( + Effect.fail(new SandboxManagerError({ message: "artifact directory io" })), + ), + ), + // Called for every ready/paused thread right after the sweep -- + // reaching it proves the sweep failure was contained. + sampleUsage: () => + Deferred.succeed(continued, undefined).pipe( + Effect.andThen(Effect.fail(new SandboxManagerError({ message: "no container" }))), + ), + reconcile: () => + Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), + } as never), + ), + Layer.provide( + Layer.mock(ProjectionSnapshotQuery)({ + getSnapshot: () => + Effect.succeed({ + ...snapshot, + threads: [activeThread, stoppedThread], + } as typeof snapshot), + getThreadDetailById: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide( + Layer.mock(OrchestrationEngineService)({ + dispatch: (command) => + Effect.gen(function* () { + dispatched.push(command); + return { sequence: dispatched.length }; + }), + streamDomainEvents: Stream.empty, + }), + ), + ); + + yield* Effect.scoped( + Effect.gen(function* () { + const reactor = yield* SandboxLifecycleReactor; + yield* reactor.start(); + const protectedIds = yield* Deferred.await(swept).pipe(Effect.timeout("5 seconds")); + // The ready thread is shielded; the stopped one is fair game. + expect([...protectedIds]).toEqual([threadId]); + // The pass reached per-thread work after the sweep failed: the + // failure was logged and contained, not propagated. + yield* Deferred.await(continued).pipe(Effect.timeout("5 seconds")); + yield* reactor.drain; + }).pipe(Effect.provide(layer)), + ); + }), + ); }); diff --git a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts index a64e854d60d2..fa640d3202b3 100644 --- a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts +++ b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts @@ -712,6 +712,30 @@ export const make = Effect.gen(function* () { const expire = Effect.fn("SandboxLifecycleReactor.expire")(function* () { const snapshot = yield* snapshots.getSnapshot(); + // Piggybacks on the periodic pass rather than owning a timer: exported + // artifact sets for threads that settled long ago (or were deleted + // out-of-band) otherwise accumulate forever. Threads whose sandbox is + // still in a non-terminal lifecycle are protected regardless of age -- + // their next stop overwrites the set, and deleting it early would cost a + // re-provision its restore seed. Best-effort: a sweep failure must not + // stall expiry. + yield* Effect.suspend(() => + runtimes.sweepExpiredArtifacts( + new Set( + snapshot.threads + .filter( + (thread) => + thread.sandbox != null && + !["stopped", "expired", "deleted"].includes(thread.sandbox.lifecycle), + ) + .map((thread) => thread.id), + ), + ), + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("sandbox artifact sweep failed", { cause: Cause.pretty(cause) }), + ), + ); const now = DateTime.toEpochMillis(yield* DateTime.now); const activeSessions = new Set( (yield* providers.listSessions()).map((session) => session.threadId), diff --git a/apps/server/src/sandbox/SandboxProviderStore.test.ts b/apps/server/src/sandbox/SandboxProviderStore.test.ts index 15c74e88860f..4e3c59d1daac 100644 --- a/apps/server/src/sandbox/SandboxProviderStore.test.ts +++ b/apps/server/src/sandbox/SandboxProviderStore.test.ts @@ -101,6 +101,7 @@ const MUTATED_ENV = [ "T3_SANDBOX_PREVIEW_PROXY_IMAGE", "T3_SANDBOX_CREDENTIAL_PROXY_IMAGE", "T3_SANDBOX_STORE_MAX_BYTES", + "T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS", ] as const; const originalEnv = new Map(MUTATED_ENV.map((key) => [key, process.env[key]] as const)); @@ -289,3 +290,111 @@ describe("provider conversation store artifacts", () => { }), ); }); + +describe("expired artifact retention sweep", () => { + const artifactSet = (root: string, threadId: string, ageMs: number) => { + const name = NodeCrypto.createHash("sha256").update(threadId).digest("hex"); + // @effect-diagnostics-next-line globalDate:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time. + const mtime = new Date(Date.now() - ageMs); + for (const file of [`${name}.bundle`, `${name}.json`, `${name}.store.tar`]) { + const path = NodePath.join(root, file); + NodeFS.writeFileSync(path, "artifact", "utf8"); + NodeFS.utimesSync(path, mtime, mtime); + } + return name; + }; + const DAY_MS = 24 * 60 * 60 * 1000; + + it.effect("deletes sets past the age cap and keeps young ones", () => + Effect.gen(function* () { + const root = makeRoot(); + const old = artifactSet(root, "thread-old", 45 * DAY_MS); + const young = artifactSet(root, "thread-young", 5 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + const removed = yield* manager.sweepExpiredArtifacts(new Set()); + + expect(removed).toBe(1); + const remaining = NodeFS.readdirSync(root).sort(); + expect(remaining).toEqual([`${young}.bundle`, `${young}.json`, `${young}.store.tar`]); + expect(remaining.some((file) => file.startsWith(old))).toBe(false); + }), + ); + + it.effect("a set is as young as its newest file", () => + Effect.gen(function* () { + // Exports rename bundle, store, and manifest together; a set whose + // manifest is fresh exported recently even if an older sibling survived + // a partial overwrite. It must not be deleted piecemeal. + const root = makeRoot(); + const name = artifactSet(root, "thread-mixed", 45 * DAY_MS); + // @effect-diagnostics-next-line globalDateInEffect:off - freshens a real filesystem mtime, which the sweep reads with wall-clock time. + const now = new Date(); + NodeFS.utimesSync(NodePath.join(root, `${name}.json`), now, now); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(3); + }), + ); + + it.effect("keeps a set belonging to an active thread regardless of age", () => + Effect.gen(function* () { + // A non-terminal sandbox will overwrite its set on the next stop; in the + // meantime that set may be the seed a re-provision restores from. + const root = makeRoot(); + const kept = artifactSet(root, "thread-active", 90 * DAY_MS); + artifactSet(root, "thread-gone", 90 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + const removed = yield* manager.sweepExpiredArtifacts(new Set(["thread-active"])); + + expect(removed).toBe(1); + expect(NodeFS.readdirSync(root).sort()).toEqual([ + `${kept}.bundle`, + `${kept}.json`, + `${kept}.store.tar`, + ]); + }), + ); + + it.effect("an explicit zero disables the sweep entirely", () => + Effect.gen(function* () { + process.env.T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS = "0"; + const root = makeRoot(); + artifactSet(root, "thread-ancient", 400 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(3); + }), + ); + + it.effect("ignores in-flight export temporaries and foreign files", () => + Effect.gen(function* () { + const root = makeRoot(); + // @effect-diagnostics-next-line globalDateInEffect:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time. + const past = new Date(Date.now() - 60 * DAY_MS); + for (const file of [`.${"a".repeat(64)}.1234.bundle.tmp`, "seeds"]) { + const path = NodePath.join(root, file); + NodeFS.writeFileSync(path, "not-an-artifact", "utf8"); + NodeFS.utimesSync(path, past, past); + } + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(2); + }), + ); + + it.effect("sweeps nothing without configured artifact storage or a directory", () => + Effect.gen(function* () { + const unconfigured = makeSandboxRuntimeManager(undefined, "linux", new FakeExecutor()); + expect(yield* unconfigured.sweepExpiredArtifacts(new Set())).toBe(0); + // A root that exists in config but was never written to (no exports yet). + const missing = NodePath.join(makeRoot(), "never-created"); + const manager = makeSandboxRuntimeManager(missing, "linux", new FakeExecutor()); + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + }), + ); +}); diff --git a/apps/server/src/sandbox/SandboxRuntimeManager.ts b/apps/server/src/sandbox/SandboxRuntimeManager.ts index 4bea946495f5..91200eed555f 100644 --- a/apps/server/src/sandbox/SandboxRuntimeManager.ts +++ b/apps/server/src/sandbox/SandboxRuntimeManager.ts @@ -88,16 +88,41 @@ export function resolveSandboxDesktopMode(): "enabled" | "disabled" { /** * Ceiling on an archived provider conversation store, in bytes. * - * Exists because nothing prunes the artifact directory: a store past this size - * is skipped rather than kept forever. Measured on a real host, one long - * thread's transcript alone reached ~30MB, so the default is set well above - * ordinary threads while still bounding the pathological ones. + * The age-based artifact sweep only reclaims whole sets long after their last + * export; this bounds each set's size at write time. Measured on a real host, + * one long thread's transcript alone reached ~30MB, so the default is set well + * above ordinary threads while still bounding the pathological ones. */ export function resolveSandboxStoreMaxBytes(): number { const raw = Number.parseInt(process.env.T3_SANDBOX_STORE_MAX_BYTES?.trim() ?? "", 10); return Number.isFinite(raw) && raw > 0 ? raw : 50 * 1024 * 1024; } +/** + * Age ceiling for exported sandbox artifact sets, in seconds. + * + * Filenames are `sha256(threadId)` and every export overwrites the same set, + * so per-thread growth is already bounded -- what leaks is sets for threads + * that were deleted out-of-band or settled long ago. Defaults to 30 days: old + * enough that a returning thread almost always finds its export (deleting a + * set degrades re-provision to a plain clone and loses the provider's archived + * conversation), young enough that the directory stops growing without bound. + * An explicit `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS=0` disables the sweep; + * anything unparseable or negative falls back to the default. + */ +export function resolveSandboxArtifactMaxAgeSeconds(): number { + const raw = Number.parseInt(process.env.T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS?.trim() ?? "", 10); + return Number.isFinite(raw) && raw >= 0 ? raw : 30 * 24 * 60 * 60; +} + +/** + * Ceiling on artifact sets removed by one sweep pass. The directory is flat + * and one set is three files, so a pass is cheap -- the cap only exists so a + * pathological backlog cannot pin the sweep for minutes; the remainder goes + * on the next pass. + */ +const ARTIFACT_SWEEP_MAX_SETS = 1000; + /** One-shot credential boundary used immediately before provider process spawn. */ export function redeemSandboxProviderEnvironment( threadId: string, @@ -171,6 +196,16 @@ export interface SandboxRuntimeManagerShape { * must not outlive the thread they belong to. */ readonly removeThreadArtifacts: (threadId: string) => Effect.Effect; + /** + * Deletes exported artifact sets whose newest file is older than + * `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS`, except sets belonging to + * `protectedThreadIds` -- threads whose sandbox is still in a non-terminal + * lifecycle, whose next stop will overwrite the set anyway. Returns the + * number of sets removed. + */ + readonly sweepExpiredArtifacts: ( + protectedThreadIds: ReadonlySet, + ) => Effect.Effect; } export class SandboxManagerError extends Schema.TaggedErrorClass()( @@ -736,6 +771,78 @@ export const makeSandboxRuntimeManager = ( ), ); }), + sweepExpiredArtifacts: Effect.fn("SandboxRuntimeManager.sweepExpiredArtifacts")( + function* (protectedThreadIds) { + const maxAgeSeconds = resolveSandboxArtifactMaxAgeSeconds(); + if (maxAgeSeconds === 0 || artifactRoot === undefined) return 0; + const protectedNames = new Set( + [...protectedThreadIds].map((threadId) => + NodeCrypto.createHash("sha256").update(threadId).digest("hex"), + ), + ); + const { removed, capped } = yield* attempt(async () => { + const entries = await NodeFSP.readdir(artifactRoot).catch((cause: unknown) => { + // No directory means no exports have happened yet: nothing to sweep. + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return [] as string[]; + throw cause; + }); + // One set per thread: `.{bundle,json,store.tar}`. + // Dot-prefixed names are in-flight temporaries owned by `exportBranch` + // and never eligible. + const sets = new Map(); + for (const entry of entries) { + const match = /^([a-f0-9]{64})\.(?:bundle|json|store\.tar)$/.exec(entry); + if (match?.[1] === undefined) continue; + const files = sets.get(match[1]) ?? []; + files.push(entry); + sets.set(match[1], files); + } + // @effect-diagnostics-next-line globalDate:off - ages are real filesystem mtimes; the virtual test clock would misdate them. + const cutoff = Date.now() - maxAgeSeconds * 1000; + let removedCount = 0; + let cappedAtLimit = false; + for (const [name, files] of sets) { + if (protectedNames.has(name)) continue; + // The newest file's mtime dates the set as a whole: exports rename + // all three files together, so a set with one young file is a set + // that exported recently. + const times = await Promise.all( + files.map((file) => + NodeFSP.stat(NodePath.resolve(artifactRoot, file)) + .then((stat) => stat.mtimeMs) + // A file deleted mid-sweep (thread deletion, another export) + // simply no longer dates the set. + .catch(() => undefined), + ), + ); + const newest = Math.max(...times.map((time) => time ?? Number.NEGATIVE_INFINITY)); + if (!Number.isFinite(newest) || newest > cutoff) continue; + if (removedCount >= ARTIFACT_SWEEP_MAX_SETS) { + cappedAtLimit = true; + break; + } + await Promise.all( + files.map((file) => + NodeFSP.rm(NodePath.resolve(artifactRoot, file), { force: true }), + ), + ); + removedCount += 1; + } + return { removed: removedCount, capped: cappedAtLimit }; + }); + if (capped) + yield* Effect.logWarning("sandbox artifact sweep hit its per-run deletion cap", { + removed, + cap: ARTIFACT_SWEEP_MAX_SETS, + }); + if (removed > 0) + yield* Effect.logInfo("sandbox artifact sweep removed expired export sets", { + removed, + maxAgeSeconds, + }); + return removed; + }, + ), }; }; From 17a02145ec4f842396ad8d3a1fdfa7f0b67321a2 Mon Sep 17 00:00:00 2001 From: awtprod Date: Fri, 21 Aug 2026 07:50:59 +0000 Subject: [PATCH 2/3] fix(server): stop the export manifest advertising an unservable store file The manifest listed store: .store.tar, but the artifact HTTP route serves only bundle|manifest -- a client following the manifest got a 404 on an advertised file. The store is transcript data used server-side for restore, so rather than serving it, mark it storeServed: false in the manifest. The manifest is written as plain JSON and its only schema-validated reader is the restore path, which ignores unknown fields. Co-Authored-By: Claude Fable 5 --- apps/server/src/sandbox/SandboxProviderStore.test.ts | 4 ++++ apps/server/src/sandbox/SandboxRuntimeManager.ts | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/server/src/sandbox/SandboxProviderStore.test.ts b/apps/server/src/sandbox/SandboxProviderStore.test.ts index 4e3c59d1daac..62ebbfb4e8ff 100644 --- a/apps/server/src/sandbox/SandboxProviderStore.test.ts +++ b/apps/server/src/sandbox/SandboxProviderStore.test.ts @@ -150,10 +150,14 @@ describe("provider conversation store artifacts", () => { NodeFS.readFileSync(NodePath.join(root, `${ARTIFACT_ID}.json`), "utf8"), ) as { store?: string; + storeServed?: boolean; storeSha256?: string; storeBytes?: number; }; expect(manifest.store).toBe(`${ARTIFACT_ID}.store.tar`); + // The artifact HTTP route serves only `bundle` and `manifest`; the + // manifest must say so rather than advertise a 404. + expect(manifest.storeServed).toBe(false); expect(manifest.storeSha256).toBe(exported.storeSha256); expect(manifest.storeBytes).toBe(STORE_CONTENTS.length); }), diff --git a/apps/server/src/sandbox/SandboxRuntimeManager.ts b/apps/server/src/sandbox/SandboxRuntimeManager.ts index 91200eed555f..93517a35c08c 100644 --- a/apps/server/src/sandbox/SandboxRuntimeManager.ts +++ b/apps/server/src/sandbox/SandboxRuntimeManager.ts @@ -670,9 +670,14 @@ export const makeSandboxRuntimeManager = ( threadId, bundle: `${name}.bundle`, bundleSha256, + // `storeServed: false` marks the store as a server-internal + // artifact: re-provision reads it from disk to restore the + // provider's conversation, but the artifact HTTP route serves + // only `bundle` and `manifest` -- a client following the + // manifest must not treat `store` as downloadable. ...(storeSha256 === undefined ? {} - : { store: `${name}.store.tar`, storeSha256, storeBytes }), + : { store: `${name}.store.tar`, storeServed: false, storeSha256, storeBytes }), ...result, }), { mode: 0o600, flag: "wx" }, From c2f2aed1f9d12b0e5993d358118809a70ddddace Mon Sep 17 00:00:00 2001 From: awtprod Date: Fri, 21 Aug 2026 07:51:03 +0000 Subject: [PATCH 3/3] docs(operations): document the sandbox runtime env flags and lifecycle The sandbox env flags existed only as code comments spread across the server. Add the sandbox block to .env.example (all flags with defaults, including the new artifact-retention cap) and an operator-focused docs/operations/sandbox-runtime.md covering the two-flag enable gate, the provision/stop/expire/re-provision lifecycle, export artifacts and where they live, the retention policy, and the store's exclusion from HTTP serving. Linked from sandbox-host.md, which stays host-bootstrap-only. Co-Authored-By: Claude Fable 5 --- .env.example | 21 +++++++++ docs/operations/sandbox-host.md | 4 +- docs/operations/sandbox-runtime.md | 74 ++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 docs/operations/sandbox-runtime.md diff --git a/.env.example b/.env.example index fc67dcef9478..66c13ab13c70 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,24 @@ T3CODE_RELAY_URL=https://relay.t3.codes # T3CODE_MOBILE_OTLP_TRACES_URL=https://api.axiom.co/v1/traces # T3CODE_MOBILE_OTLP_TRACES_DATASET=t3-code-mobile-traces-dev # T3CODE_MOBILE_OTLP_TRACES_TOKEN=xaat-... + +# Optional: per-thread container sandboxing (server runtime, all off by default). +# See docs/operations/sandbox-runtime.md for lifecycle and retention details. +# Master switch: digest-pinned workspace image; unset = threads run on the host. +# T3_SANDBOX_IMAGE=registry.example/t3-sandbox@sha256:... +# Required alongside T3_SANDBOX_IMAGE: preview/signaling sidecar image. +# T3_SANDBOX_PREVIEW_PROXY_IMAGE=registry.example/t3-preview-proxy@sha256:... +# Opt-in outbound egress sidecar; unset = sandboxes have no egress. +# T3_SANDBOX_EGRESS_PROXY_IMAGE=registry.example/t3-egress-proxy@sha256:... +# Comma-separated sha256 digests of cache volumes the server will mount. +# T3_SANDBOX_TRUSTED_CACHE_DIGESTS= +# Override for the export artifact directory; defaults to /sandbox-artifacts. +# T3_SANDBOX_ARTIFACT_DIR= +# Ceiling on an archived provider conversation store, in bytes (default 50MB). +# T3_SANDBOX_STORE_MAX_BYTES=52428800 +# Age cap on exported artifact sets, in seconds (default 30 days; 0 disables the sweep). +# T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS=2592000 +# "disabled" runs sandboxes headless: no X server, no desktop streaming. +# T3_SANDBOX_DESKTOP=disabled +# "disabled" omits container/volume storage quotas on hosts without XFS pquota. +# T3_SANDBOX_CONTAINER_STORAGE_QUOTA=disabled diff --git a/docs/operations/sandbox-host.md b/docs/operations/sandbox-host.md index fcb08b767b79..623e6af9cc52 100644 --- a/docs/operations/sandbox-host.md +++ b/docs/operations/sandbox-host.md @@ -6,7 +6,9 @@ it needs is not installed, and the environment variables that switch it on are not set. This runbook covers making the runtime available, proving it works, switching -sandboxing on deliberately, and switching it back off. +sandboxing on deliberately, and switching it back off. What the server does +with the runtime once it exists — lifecycle, export artifacts, retention — is +covered separately in [sandbox-runtime.md](./sandbox-runtime.md). Everything root-required lives in one reviewed script, `deploy/openclaw/sandbox/bootstrap-sandbox-host.sh`. The service account diff --git a/docs/operations/sandbox-runtime.md b/docs/operations/sandbox-runtime.md new file mode 100644 index 000000000000..93b69581ccf0 --- /dev/null +++ b/docs/operations/sandbox-runtime.md @@ -0,0 +1,74 @@ +# Sandbox runtime operation + +Host bootstrap (podman, quotas, networking) is covered by +[sandbox-host.md](./sandbox-host.md). This page covers what the server does +once the runtime exists: how sandboxing switches on, how a thread's sandbox +lives and dies, and where its exported data goes. + +## Enabling + +Two variables gate everything, both digest-pinned images: + +- `T3_SANDBOX_IMAGE` — the workspace image. Unset means sandboxing is off and + threads run directly on the host, exactly as before. A project's `.t3` file + `sandbox.image` overrides it per project. +- `T3_SANDBOX_PREVIEW_PROXY_IMAGE` — the preview/signaling sidecar, required + whenever the workspace image is set. With the image set but this missing, + provisioning refuses and the thread gets a "sandbox isolation is disabled" + notice instead of a container. + +Everything else is optional: `T3_SANDBOX_EGRESS_PROXY_IMAGE` (no egress +without it), `T3_SANDBOX_CREDENTIAL_PROXY_IMAGE`, `T3_SANDBOX_DESKTOP=disabled` +for headless hosts, `T3_SANDBOX_CONTAINER_STORAGE_QUOTA=disabled` for hosts +that cannot enforce XFS project quotas. The full list with defaults is in the +repository root `.env.example`. + +## Lifecycle + +A sandboxed thread's container moves through `provisioning → ready`, then +eventually `stopping → stopped` (user- or settle-driven) or `expired` (the +server's periodic pass, which runs every minute, enforces the idle timeout and +maximum lifetime). A stop or expiry first **exports** the thread's work, then +tears down the container, its sidecars, network, and volumes. + +A later turn on a stopped or expired thread **re-provisions from the export**: +the branch bundle seeds the repository at the exported head commit, and the +archived provider conversation store restores the agent's context. A missing +or tampered artifact degrades this to a plain clone at the recorded base +commit — the thread still works, but the previous session's commits and the +agent's memory of the conversation are gone. + +A server restart deliberately fail-closes running sandboxes; affected threads +re-provision on their next turn (see sandbox-host.md, Known caveats). + +## Export artifacts + +Each export writes one artifact set to `/sandbox-artifacts/` +(overridable with `T3_SANDBOX_ARTIFACT_DIR`), named `sha256(threadId)`: + +- `.bundle` — git bundle of the thread branch. +- `.json` — manifest: digests, sizes, branch/commit provenance. +- `.store.tar` — the provider's archived conversation store, capped at + `T3_SANDBOX_STORE_MAX_BYTES` (default 50MB; an oversized store is skipped, + the branch still exports). + +Every export for a thread overwrites the same set, so per-thread disk use is +bounded. The bundle and manifest are downloadable at +`/api/sandbox-artifacts//bundle|manifest`. The store is deliberately **not +served over HTTP** — it is transcript data used server-side for restore only, +and the manifest marks it `storeServed: false`. + +## Retention + +- **Thread deletion** removes the thread's artifact set immediately, along + with its container if one is running: transcripts and commits must not + outlive the thread. +- **Age sweep**: the same periodic pass that expires idle sandboxes deletes + artifact sets whose newest file is older than + `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS` (default 30 days; set `0` to disable). + Sets belonging to threads whose sandbox is in a non-terminal lifecycle are + kept regardless of age. The 30-day default trades disk growth against + restore quality: a thread revived after the cap still provisions, but from a + plain clone, without the previous session's commits or agent memory. Raise + the cap (or disable the sweep) on hosts where very long-dormant threads are + expected back.