diff --git a/.env.example b/.env.example index fc67dcef9478..66c13ab13c70 100644 --- a/.env.example +++ b/.env.example @@ -29,3 +29,24 @@ T3CODE_RELAY_URL=https://relay.t3.codes # T3CODE_MOBILE_OTLP_TRACES_URL=https://api.axiom.co/v1/traces # T3CODE_MOBILE_OTLP_TRACES_DATASET=t3-code-mobile-traces-dev # T3CODE_MOBILE_OTLP_TRACES_TOKEN=xaat-... + +# Optional: per-thread container sandboxing (server runtime, all off by default). +# See docs/operations/sandbox-runtime.md for lifecycle and retention details. +# Master switch: digest-pinned workspace image; unset = threads run on the host. +# T3_SANDBOX_IMAGE=registry.example/t3-sandbox@sha256:... +# Required alongside T3_SANDBOX_IMAGE: preview/signaling sidecar image. +# T3_SANDBOX_PREVIEW_PROXY_IMAGE=registry.example/t3-preview-proxy@sha256:... +# Opt-in outbound egress sidecar; unset = sandboxes have no egress. +# T3_SANDBOX_EGRESS_PROXY_IMAGE=registry.example/t3-egress-proxy@sha256:... +# Comma-separated sha256 digests of cache volumes the server will mount. +# T3_SANDBOX_TRUSTED_CACHE_DIGESTS= +# Override for the export artifact directory; defaults to /sandbox-artifacts. +# T3_SANDBOX_ARTIFACT_DIR= +# Ceiling on an archived provider conversation store, in bytes (default 50MB). +# T3_SANDBOX_STORE_MAX_BYTES=52428800 +# Age cap on exported artifact sets, in seconds (default 30 days; 0 disables the sweep). +# T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS=2592000 +# "disabled" runs sandboxes headless: no X server, no desktop streaming. +# T3_SANDBOX_DESKTOP=disabled +# "disabled" omits container/volume storage quotas on hosts without XFS pquota. +# T3_SANDBOX_CONTAINER_STORAGE_QUOTA=disabled diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index 2edbc6e27a51..164659b732cc 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -491,6 +491,7 @@ export const makeOrchestrationIntegrationHarness = ( recoverPreview: () => Effect.die("recoverPreview should not be called in this test"), revokeCredentials: () => Effect.succeed(0), removeThreadArtifacts: () => Effect.void, + sweepExpiredArtifacts: () => Effect.succeed(0), }); } const projectFileLoaderLayer = Layer.succeed(T3ProjectFileLoader, { diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 91a6628dabe3..a62d0d3b29f2 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -496,6 +496,7 @@ describe("ProviderCommandReactor", () => { recoverPreview: () => Effect.succeed(false), revokeCredentials: () => Effect.succeed(0), removeThreadArtifacts: () => Effect.void, + sweepExpiredArtifacts: () => Effect.succeed(0), } satisfies SandboxRuntimeManagerShape), ), // The real directory over the in-memory database: the reactor clears a diff --git a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts index b5d8516c9dc7..064ef01c4979 100644 --- a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts +++ b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.test.ts @@ -140,6 +140,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision, reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -249,6 +250,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision: () => Effect.die("runtime must not run without an image"), reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -340,6 +342,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), exportBranch: () => Effect.succeed({ commit: "b".repeat(40), @@ -441,6 +444,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), stop: () => Effect.die("no container exists for an unprovisioned sandbox"), exportBranch: () => Effect.die("nothing to export without a container"), } as never), @@ -540,6 +544,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), exportBranch: () => Effect.succeed({ commit: "b".repeat(40), @@ -643,6 +648,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), provision: () => Effect.die("provisioning is not reached on this path"), reconcile: () => Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), @@ -742,6 +748,7 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { ), Layer.provide( Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: () => Effect.succeed(0), stop: (_runtime: string, id: string) => Effect.sync(() => { stopped.push(id); @@ -787,4 +794,107 @@ it.layer(NodeServices.layer)("manual sandbox lifecycle provisioning", (it) => { expect(dispatched.map((command) => command.type)).toEqual(["sandbox.stop.complete"]); }), ); + + it.effect( + "the periodic pass sweeps expired artifacts, shields active threads, and survives a sweep failure", + () => + Effect.gen(function* () { + // The sweep rides the same periodic pass as expiry. Threads whose + // sandbox is in a non-terminal lifecycle must arrive in the protected + // set (their artifact set may seed a re-provision), terminal ones must + // not -- and a failing sweep must not stall the rest of the pass. + const swept = yield* Deferred.make>(); + const continued = yield* Deferred.make(); + const dispatched: OrchestrationCommand[] = []; + const OLD = "2026-08-15T00:00:00.000Z"; + const sandboxOf = (lifecycle: "ready" | "stopped") => ({ + lifecycle, + runtime: "podman" as const, + branch: { branchName: "t3/thread/x", baseCommit: "a".repeat(40) }, + limits: { + cpuCount: 2, + memoryBytes: 4_294_967_296, + diskBytes: 21_474_836_480, + processCount: 512, + idleTimeoutSeconds: 60, + maximumLifetimeSeconds: 60, + }, + desktop: { status: "unavailable" as const }, + services: [], + controller: { kind: "none" as const }, + createdAt: OLD, + lastActiveAt: OLD, + }); + const activeThread = { ...snapshot.threads[0]!, sandbox: sandboxOf("ready") }; + const stoppedThread = { + ...snapshot.threads[0]!, + id: ThreadId.make("thread-stopped"), + sandbox: sandboxOf("stopped"), + }; + const layer = Layer.effect(SandboxLifecycleReactor, make).pipe( + Layer.provide(NodeServices.layer), + Layer.provide( + Layer.mock(GitWorkflowService)({ + localStatus: () => Effect.succeed({ isRepo: true, refName: "main" } as never), + }), + ), + Layer.provide(Layer.mock(ProviderService)({ listSessions: () => Effect.succeed([]) })), + Layer.provide( + Layer.succeed(T3ProjectFileLoader, { load: () => Effect.succeed(Option.none()) }), + ), + Layer.provide( + Layer.succeed(SandboxRuntimeManager, { + sweepExpiredArtifacts: (protectedThreadIds: ReadonlySet) => + Deferred.succeed(swept, protectedThreadIds).pipe( + Effect.andThen( + Effect.fail(new SandboxManagerError({ message: "artifact directory io" })), + ), + ), + // Called for every ready/paused thread right after the sweep -- + // reaching it proves the sweep failure was contained. + sampleUsage: () => + Deferred.succeed(continued, undefined).pipe( + Effect.andThen(Effect.fail(new SandboxManagerError({ message: "no container" }))), + ), + reconcile: () => + Effect.succeed({ activeThreadIds: [], missingThreadIds: [], orphanThreadIds: [] }), + } as never), + ), + Layer.provide( + Layer.mock(ProjectionSnapshotQuery)({ + getSnapshot: () => + Effect.succeed({ + ...snapshot, + threads: [activeThread, stoppedThread], + } as typeof snapshot), + getThreadDetailById: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide( + Layer.mock(OrchestrationEngineService)({ + dispatch: (command) => + Effect.gen(function* () { + dispatched.push(command); + return { sequence: dispatched.length }; + }), + streamDomainEvents: Stream.empty, + }), + ), + ); + + yield* Effect.scoped( + Effect.gen(function* () { + const reactor = yield* SandboxLifecycleReactor; + yield* reactor.start(); + const protectedIds = yield* Deferred.await(swept).pipe(Effect.timeout("5 seconds")); + // The ready thread is shielded; the stopped one is fair game. + expect([...protectedIds]).toEqual([threadId]); + // The pass reached per-thread work after the sweep failed: the + // failure was logged and contained, not propagated. + yield* Deferred.await(continued).pipe(Effect.timeout("5 seconds")); + yield* reactor.drain; + }).pipe(Effect.provide(layer)), + ); + }), + ); }); diff --git a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts index a64e854d60d2..fa640d3202b3 100644 --- a/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts +++ b/apps/server/src/orchestration/Layers/SandboxLifecycleReactor.ts @@ -712,6 +712,30 @@ export const make = Effect.gen(function* () { const expire = Effect.fn("SandboxLifecycleReactor.expire")(function* () { const snapshot = yield* snapshots.getSnapshot(); + // Piggybacks on the periodic pass rather than owning a timer: exported + // artifact sets for threads that settled long ago (or were deleted + // out-of-band) otherwise accumulate forever. Threads whose sandbox is + // still in a non-terminal lifecycle are protected regardless of age -- + // their next stop overwrites the set, and deleting it early would cost a + // re-provision its restore seed. Best-effort: a sweep failure must not + // stall expiry. + yield* Effect.suspend(() => + runtimes.sweepExpiredArtifacts( + new Set( + snapshot.threads + .filter( + (thread) => + thread.sandbox != null && + !["stopped", "expired", "deleted"].includes(thread.sandbox.lifecycle), + ) + .map((thread) => thread.id), + ), + ), + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("sandbox artifact sweep failed", { cause: Cause.pretty(cause) }), + ), + ); const now = DateTime.toEpochMillis(yield* DateTime.now); const activeSessions = new Set( (yield* providers.listSessions()).map((session) => session.threadId), diff --git a/apps/server/src/sandbox/SandboxProviderStore.test.ts b/apps/server/src/sandbox/SandboxProviderStore.test.ts index 15c74e88860f..62ebbfb4e8ff 100644 --- a/apps/server/src/sandbox/SandboxProviderStore.test.ts +++ b/apps/server/src/sandbox/SandboxProviderStore.test.ts @@ -101,6 +101,7 @@ const MUTATED_ENV = [ "T3_SANDBOX_PREVIEW_PROXY_IMAGE", "T3_SANDBOX_CREDENTIAL_PROXY_IMAGE", "T3_SANDBOX_STORE_MAX_BYTES", + "T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS", ] as const; const originalEnv = new Map(MUTATED_ENV.map((key) => [key, process.env[key]] as const)); @@ -149,10 +150,14 @@ describe("provider conversation store artifacts", () => { NodeFS.readFileSync(NodePath.join(root, `${ARTIFACT_ID}.json`), "utf8"), ) as { store?: string; + storeServed?: boolean; storeSha256?: string; storeBytes?: number; }; expect(manifest.store).toBe(`${ARTIFACT_ID}.store.tar`); + // The artifact HTTP route serves only `bundle` and `manifest`; the + // manifest must say so rather than advertise a 404. + expect(manifest.storeServed).toBe(false); expect(manifest.storeSha256).toBe(exported.storeSha256); expect(manifest.storeBytes).toBe(STORE_CONTENTS.length); }), @@ -289,3 +294,111 @@ describe("provider conversation store artifacts", () => { }), ); }); + +describe("expired artifact retention sweep", () => { + const artifactSet = (root: string, threadId: string, ageMs: number) => { + const name = NodeCrypto.createHash("sha256").update(threadId).digest("hex"); + // @effect-diagnostics-next-line globalDate:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time. + const mtime = new Date(Date.now() - ageMs); + for (const file of [`${name}.bundle`, `${name}.json`, `${name}.store.tar`]) { + const path = NodePath.join(root, file); + NodeFS.writeFileSync(path, "artifact", "utf8"); + NodeFS.utimesSync(path, mtime, mtime); + } + return name; + }; + const DAY_MS = 24 * 60 * 60 * 1000; + + it.effect("deletes sets past the age cap and keeps young ones", () => + Effect.gen(function* () { + const root = makeRoot(); + const old = artifactSet(root, "thread-old", 45 * DAY_MS); + const young = artifactSet(root, "thread-young", 5 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + const removed = yield* manager.sweepExpiredArtifacts(new Set()); + + expect(removed).toBe(1); + const remaining = NodeFS.readdirSync(root).sort(); + expect(remaining).toEqual([`${young}.bundle`, `${young}.json`, `${young}.store.tar`]); + expect(remaining.some((file) => file.startsWith(old))).toBe(false); + }), + ); + + it.effect("a set is as young as its newest file", () => + Effect.gen(function* () { + // Exports rename bundle, store, and manifest together; a set whose + // manifest is fresh exported recently even if an older sibling survived + // a partial overwrite. It must not be deleted piecemeal. + const root = makeRoot(); + const name = artifactSet(root, "thread-mixed", 45 * DAY_MS); + // @effect-diagnostics-next-line globalDateInEffect:off - freshens a real filesystem mtime, which the sweep reads with wall-clock time. + const now = new Date(); + NodeFS.utimesSync(NodePath.join(root, `${name}.json`), now, now); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(3); + }), + ); + + it.effect("keeps a set belonging to an active thread regardless of age", () => + Effect.gen(function* () { + // A non-terminal sandbox will overwrite its set on the next stop; in the + // meantime that set may be the seed a re-provision restores from. + const root = makeRoot(); + const kept = artifactSet(root, "thread-active", 90 * DAY_MS); + artifactSet(root, "thread-gone", 90 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + const removed = yield* manager.sweepExpiredArtifacts(new Set(["thread-active"])); + + expect(removed).toBe(1); + expect(NodeFS.readdirSync(root).sort()).toEqual([ + `${kept}.bundle`, + `${kept}.json`, + `${kept}.store.tar`, + ]); + }), + ); + + it.effect("an explicit zero disables the sweep entirely", () => + Effect.gen(function* () { + process.env.T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS = "0"; + const root = makeRoot(); + artifactSet(root, "thread-ancient", 400 * DAY_MS); + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(3); + }), + ); + + it.effect("ignores in-flight export temporaries and foreign files", () => + Effect.gen(function* () { + const root = makeRoot(); + // @effect-diagnostics-next-line globalDateInEffect:off - backdates real filesystem mtimes, which the sweep reads with wall-clock time. + const past = new Date(Date.now() - 60 * DAY_MS); + for (const file of [`.${"a".repeat(64)}.1234.bundle.tmp`, "seeds"]) { + const path = NodePath.join(root, file); + NodeFS.writeFileSync(path, "not-an-artifact", "utf8"); + NodeFS.utimesSync(path, past, past); + } + const manager = makeSandboxRuntimeManager(root, "linux", new FakeExecutor()); + + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + expect(NodeFS.readdirSync(root)).toHaveLength(2); + }), + ); + + it.effect("sweeps nothing without configured artifact storage or a directory", () => + Effect.gen(function* () { + const unconfigured = makeSandboxRuntimeManager(undefined, "linux", new FakeExecutor()); + expect(yield* unconfigured.sweepExpiredArtifacts(new Set())).toBe(0); + // A root that exists in config but was never written to (no exports yet). + const missing = NodePath.join(makeRoot(), "never-created"); + const manager = makeSandboxRuntimeManager(missing, "linux", new FakeExecutor()); + expect(yield* manager.sweepExpiredArtifacts(new Set())).toBe(0); + }), + ); +}); diff --git a/apps/server/src/sandbox/SandboxRuntimeManager.ts b/apps/server/src/sandbox/SandboxRuntimeManager.ts index 4bea946495f5..93517a35c08c 100644 --- a/apps/server/src/sandbox/SandboxRuntimeManager.ts +++ b/apps/server/src/sandbox/SandboxRuntimeManager.ts @@ -88,16 +88,41 @@ export function resolveSandboxDesktopMode(): "enabled" | "disabled" { /** * Ceiling on an archived provider conversation store, in bytes. * - * Exists because nothing prunes the artifact directory: a store past this size - * is skipped rather than kept forever. Measured on a real host, one long - * thread's transcript alone reached ~30MB, so the default is set well above - * ordinary threads while still bounding the pathological ones. + * The age-based artifact sweep only reclaims whole sets long after their last + * export; this bounds each set's size at write time. Measured on a real host, + * one long thread's transcript alone reached ~30MB, so the default is set well + * above ordinary threads while still bounding the pathological ones. */ export function resolveSandboxStoreMaxBytes(): number { const raw = Number.parseInt(process.env.T3_SANDBOX_STORE_MAX_BYTES?.trim() ?? "", 10); return Number.isFinite(raw) && raw > 0 ? raw : 50 * 1024 * 1024; } +/** + * Age ceiling for exported sandbox artifact sets, in seconds. + * + * Filenames are `sha256(threadId)` and every export overwrites the same set, + * so per-thread growth is already bounded -- what leaks is sets for threads + * that were deleted out-of-band or settled long ago. Defaults to 30 days: old + * enough that a returning thread almost always finds its export (deleting a + * set degrades re-provision to a plain clone and loses the provider's archived + * conversation), young enough that the directory stops growing without bound. + * An explicit `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS=0` disables the sweep; + * anything unparseable or negative falls back to the default. + */ +export function resolveSandboxArtifactMaxAgeSeconds(): number { + const raw = Number.parseInt(process.env.T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS?.trim() ?? "", 10); + return Number.isFinite(raw) && raw >= 0 ? raw : 30 * 24 * 60 * 60; +} + +/** + * Ceiling on artifact sets removed by one sweep pass. The directory is flat + * and one set is three files, so a pass is cheap -- the cap only exists so a + * pathological backlog cannot pin the sweep for minutes; the remainder goes + * on the next pass. + */ +const ARTIFACT_SWEEP_MAX_SETS = 1000; + /** One-shot credential boundary used immediately before provider process spawn. */ export function redeemSandboxProviderEnvironment( threadId: string, @@ -171,6 +196,16 @@ export interface SandboxRuntimeManagerShape { * must not outlive the thread they belong to. */ readonly removeThreadArtifacts: (threadId: string) => Effect.Effect; + /** + * Deletes exported artifact sets whose newest file is older than + * `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS`, except sets belonging to + * `protectedThreadIds` -- threads whose sandbox is still in a non-terminal + * lifecycle, whose next stop will overwrite the set anyway. Returns the + * number of sets removed. + */ + readonly sweepExpiredArtifacts: ( + protectedThreadIds: ReadonlySet, + ) => Effect.Effect; } export class SandboxManagerError extends Schema.TaggedErrorClass()( @@ -635,9 +670,14 @@ export const makeSandboxRuntimeManager = ( threadId, bundle: `${name}.bundle`, bundleSha256, + // `storeServed: false` marks the store as a server-internal + // artifact: re-provision reads it from disk to restore the + // provider's conversation, but the artifact HTTP route serves + // only `bundle` and `manifest` -- a client following the + // manifest must not treat `store` as downloadable. ...(storeSha256 === undefined ? {} - : { store: `${name}.store.tar`, storeSha256, storeBytes }), + : { store: `${name}.store.tar`, storeServed: false, storeSha256, storeBytes }), ...result, }), { mode: 0o600, flag: "wx" }, @@ -736,6 +776,78 @@ export const makeSandboxRuntimeManager = ( ), ); }), + sweepExpiredArtifacts: Effect.fn("SandboxRuntimeManager.sweepExpiredArtifacts")( + function* (protectedThreadIds) { + const maxAgeSeconds = resolveSandboxArtifactMaxAgeSeconds(); + if (maxAgeSeconds === 0 || artifactRoot === undefined) return 0; + const protectedNames = new Set( + [...protectedThreadIds].map((threadId) => + NodeCrypto.createHash("sha256").update(threadId).digest("hex"), + ), + ); + const { removed, capped } = yield* attempt(async () => { + const entries = await NodeFSP.readdir(artifactRoot).catch((cause: unknown) => { + // No directory means no exports have happened yet: nothing to sweep. + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return [] as string[]; + throw cause; + }); + // One set per thread: `.{bundle,json,store.tar}`. + // Dot-prefixed names are in-flight temporaries owned by `exportBranch` + // and never eligible. + const sets = new Map(); + for (const entry of entries) { + const match = /^([a-f0-9]{64})\.(?:bundle|json|store\.tar)$/.exec(entry); + if (match?.[1] === undefined) continue; + const files = sets.get(match[1]) ?? []; + files.push(entry); + sets.set(match[1], files); + } + // @effect-diagnostics-next-line globalDate:off - ages are real filesystem mtimes; the virtual test clock would misdate them. + const cutoff = Date.now() - maxAgeSeconds * 1000; + let removedCount = 0; + let cappedAtLimit = false; + for (const [name, files] of sets) { + if (protectedNames.has(name)) continue; + // The newest file's mtime dates the set as a whole: exports rename + // all three files together, so a set with one young file is a set + // that exported recently. + const times = await Promise.all( + files.map((file) => + NodeFSP.stat(NodePath.resolve(artifactRoot, file)) + .then((stat) => stat.mtimeMs) + // A file deleted mid-sweep (thread deletion, another export) + // simply no longer dates the set. + .catch(() => undefined), + ), + ); + const newest = Math.max(...times.map((time) => time ?? Number.NEGATIVE_INFINITY)); + if (!Number.isFinite(newest) || newest > cutoff) continue; + if (removedCount >= ARTIFACT_SWEEP_MAX_SETS) { + cappedAtLimit = true; + break; + } + await Promise.all( + files.map((file) => + NodeFSP.rm(NodePath.resolve(artifactRoot, file), { force: true }), + ), + ); + removedCount += 1; + } + return { removed: removedCount, capped: cappedAtLimit }; + }); + if (capped) + yield* Effect.logWarning("sandbox artifact sweep hit its per-run deletion cap", { + removed, + cap: ARTIFACT_SWEEP_MAX_SETS, + }); + if (removed > 0) + yield* Effect.logInfo("sandbox artifact sweep removed expired export sets", { + removed, + maxAgeSeconds, + }); + return removed; + }, + ), }; }; diff --git a/docs/operations/sandbox-host.md b/docs/operations/sandbox-host.md index fcb08b767b79..623e6af9cc52 100644 --- a/docs/operations/sandbox-host.md +++ b/docs/operations/sandbox-host.md @@ -6,7 +6,9 @@ it needs is not installed, and the environment variables that switch it on are not set. This runbook covers making the runtime available, proving it works, switching -sandboxing on deliberately, and switching it back off. +sandboxing on deliberately, and switching it back off. What the server does +with the runtime once it exists — lifecycle, export artifacts, retention — is +covered separately in [sandbox-runtime.md](./sandbox-runtime.md). Everything root-required lives in one reviewed script, `deploy/openclaw/sandbox/bootstrap-sandbox-host.sh`. The service account diff --git a/docs/operations/sandbox-runtime.md b/docs/operations/sandbox-runtime.md new file mode 100644 index 000000000000..93b69581ccf0 --- /dev/null +++ b/docs/operations/sandbox-runtime.md @@ -0,0 +1,74 @@ +# Sandbox runtime operation + +Host bootstrap (podman, quotas, networking) is covered by +[sandbox-host.md](./sandbox-host.md). This page covers what the server does +once the runtime exists: how sandboxing switches on, how a thread's sandbox +lives and dies, and where its exported data goes. + +## Enabling + +Two variables gate everything, both digest-pinned images: + +- `T3_SANDBOX_IMAGE` — the workspace image. Unset means sandboxing is off and + threads run directly on the host, exactly as before. A project's `.t3` file + `sandbox.image` overrides it per project. +- `T3_SANDBOX_PREVIEW_PROXY_IMAGE` — the preview/signaling sidecar, required + whenever the workspace image is set. With the image set but this missing, + provisioning refuses and the thread gets a "sandbox isolation is disabled" + notice instead of a container. + +Everything else is optional: `T3_SANDBOX_EGRESS_PROXY_IMAGE` (no egress +without it), `T3_SANDBOX_CREDENTIAL_PROXY_IMAGE`, `T3_SANDBOX_DESKTOP=disabled` +for headless hosts, `T3_SANDBOX_CONTAINER_STORAGE_QUOTA=disabled` for hosts +that cannot enforce XFS project quotas. The full list with defaults is in the +repository root `.env.example`. + +## Lifecycle + +A sandboxed thread's container moves through `provisioning → ready`, then +eventually `stopping → stopped` (user- or settle-driven) or `expired` (the +server's periodic pass, which runs every minute, enforces the idle timeout and +maximum lifetime). A stop or expiry first **exports** the thread's work, then +tears down the container, its sidecars, network, and volumes. + +A later turn on a stopped or expired thread **re-provisions from the export**: +the branch bundle seeds the repository at the exported head commit, and the +archived provider conversation store restores the agent's context. A missing +or tampered artifact degrades this to a plain clone at the recorded base +commit — the thread still works, but the previous session's commits and the +agent's memory of the conversation are gone. + +A server restart deliberately fail-closes running sandboxes; affected threads +re-provision on their next turn (see sandbox-host.md, Known caveats). + +## Export artifacts + +Each export writes one artifact set to `/sandbox-artifacts/` +(overridable with `T3_SANDBOX_ARTIFACT_DIR`), named `sha256(threadId)`: + +- `.bundle` — git bundle of the thread branch. +- `.json` — manifest: digests, sizes, branch/commit provenance. +- `.store.tar` — the provider's archived conversation store, capped at + `T3_SANDBOX_STORE_MAX_BYTES` (default 50MB; an oversized store is skipped, + the branch still exports). + +Every export for a thread overwrites the same set, so per-thread disk use is +bounded. The bundle and manifest are downloadable at +`/api/sandbox-artifacts//bundle|manifest`. The store is deliberately **not +served over HTTP** — it is transcript data used server-side for restore only, +and the manifest marks it `storeServed: false`. + +## Retention + +- **Thread deletion** removes the thread's artifact set immediately, along + with its container if one is running: transcripts and commits must not + outlive the thread. +- **Age sweep**: the same periodic pass that expires idle sandboxes deletes + artifact sets whose newest file is older than + `T3_SANDBOX_ARTIFACT_MAX_AGE_SECONDS` (default 30 days; set `0` to disable). + Sets belonging to threads whose sandbox is in a non-terminal lifecycle are + kept regardless of age. The 30-day default trades disk growth against + restore quality: a thread revived after the cap still provisions, but from a + plain clone, without the previous session's commits or agent memory. Raise + the cap (or disable the sweep) on hosts where very long-dormant threads are + expected back.