Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion src/lib/actions/sandbox/rebuild-backup-phase.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";

import * as sandboxState from "../../state/sandbox";
import {
normalizeRebuildObservabilityPolicyPresets,
normalizeRebuildTargetPolicyPresets,
Expand All @@ -11,6 +12,10 @@ import {
} from "./rebuild-backup-phase";

describe("rebuild web-search policy normalization", () => {
afterEach(() => {
vi.restoreAllMocks();
});

it("keeps only the durable Tavily provider and removes stale nous-web", () => {
expect(
normalizeRebuildWebSearchPolicyPresets(
Expand Down Expand Up @@ -85,6 +90,37 @@ describe("rebuild web-search policy normalization", () => {

expect(result?.policyPresets).toEqual([]);
expect(result?.sessionPolicyPresets).toEqual([]);
expect(result?.backupWasForceSkipped).toBe(false);
});

it("records when --force skips a total backup failure", () => {
vi.spyOn(console, "warn").mockImplementation(() => undefined);
vi.spyOn(console, "log").mockImplementation(() => undefined);
vi.spyOn(sandboxState, "backupSandboxState").mockReturnValue({
success: false,
backedUpDirs: [],
backedUpFiles: [],
failedDirs: [".openclaw"],
failedFiles: ["openclaw.json"],
});

const result = runRebuildBackupPhase({
sandboxName: "alpha",
sandboxEntry: { name: "alpha", agent: "openclaw", policies: [] },
staleRecovery: false,
preparedRecoveryManifest: null,
messagingPlan: null,
webSearchConfig: null,
force: true,
log: vi.fn(),
bail: (message): never => {
throw new Error(message);
},
relockShieldsIfNeeded: () => true,
});

expect(result?.backupManifest).toBeNull();
expect(result?.backupWasForceSkipped).toBe(true);
});

it("removes stale built-in observability egress from disabled and restricted rebuild targets", () => {
Expand Down
7 changes: 6 additions & 1 deletion src/lib/actions/sandbox/rebuild-backup-phase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@ export interface RebuildBackupPhaseInput {
preparedRecoveryManifest: RebuildBackupManifest;
messagingPlan: SandboxMessagingPlan | null;
webSearchConfig: WebSearchConfig | null;
force?: boolean;
log: RebuildLog;
bail: RebuildBail;
relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean;
}

export interface RebuildBackupPhaseResult {
backupManifest: RebuildBackupManifest;
backupWasForceSkipped: boolean;
policyPresets: string[];
sessionPolicyPresets: string[] | null;
}
Expand Down Expand Up @@ -140,8 +142,11 @@ export function runRebuildBackupPhase(
input.log,
input.relockShieldsIfNeeded,
input.bail,
{ force: input.force },
);
if (backupManifest === undefined) return null;
const backupWasForceSkipped =
input.force === true && !input.staleRecovery && backupManifest === null;

const registryPolicyPresets = Array.isArray(input.sandboxEntry.policies)
? input.sandboxEntry.policies.filter(
Expand Down Expand Up @@ -173,5 +178,5 @@ export function runRebuildBackupPhase(
true,
).policyPresets;

return { backupManifest, policyPresets, sessionPolicyPresets };
return { backupManifest, backupWasForceSkipped, policyPresets, sessionPolicyPresets };
}
88 changes: 88 additions & 0 deletions src/lib/actions/sandbox/rebuild-flow-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,94 @@ describe("rebuild agent base image preflight", () => {
});
});

describe("backupSandboxStateForRebuild with --force", () => {
let warnSpy: MockInstance;
let errorSpy: MockInstance;
let backupSpy: MockInstance;

beforeEach(() => {
warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
vi.spyOn(console, "log").mockImplementation(() => undefined);

backupSpy = vi.spyOn(sandboxState, "backupSandboxState");
});

afterEach(() => {
vi.restoreAllMocks();
});

it("returns null (skip) when backup fails completely and force is set", () => {
backupSpy.mockReturnValue({
success: false,
backedUpDirs: [],
backedUpFiles: [],
failedDirs: [".state"],
failedFiles: ["config.toml"],
manifest: null,
});
const result = backupSandboxStateForRebuild(
"alpha",
makeSandboxEntry(),
false,
() => undefined,
() => true,
makeBail(),
{ force: true },
);

expect(result).toBeNull();
const warnLines = warnSpy.mock.calls.map((args: unknown[]) => String(args[0]));
expect(warnLines.some((line: string) => line.includes("--force was specified"))).toBe(true);
});

it("aborts with hint when backup fails completely without force", () => {
backupSpy.mockReturnValue({
success: false,
backedUpDirs: [],
backedUpFiles: [],
failedDirs: [".state"],
failedFiles: [],
manifest: null,
});
expect(() =>
backupSandboxStateForRebuild(
"alpha",
makeSandboxEntry(),
false,
() => undefined,
() => true,
makeBail(),
),
).toThrow("bail: Failed to back up sandbox state.");

const errorLines = errorSpy.mock.calls.map((args: unknown[]) => String(args[0]));
expect(errorLines.some((line: string) => line.includes("rebuild --force"))).toBe(true);
});

it("aborts without force even when force option is explicitly false", () => {
backupSpy.mockReturnValue({
success: false,
backedUpDirs: [],
backedUpFiles: [],
failedDirs: [".state"],
failedFiles: [],
manifest: null,
});
expect(() =>
backupSandboxStateForRebuild(
"alpha",
makeSandboxEntry(),
false,
() => undefined,
() => true,
makeBail(),
{ force: false },
),
).toThrow("bail: Failed to back up sandbox state.");
});
});

describe("warnUnpreservedUserManagedFiles", () => {
let warnSpy: MockInstance;
let logSpy: MockInstance;
Expand Down
13 changes: 13 additions & 0 deletions src/lib/actions/sandbox/rebuild-flow-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ export function backupSandboxStateForRebuild(
log: (msg: string) => void,
relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean,
bail: (msg: string, code?: number) => never,
options?: { force?: boolean },
): sandboxState.RebuildManifest | null | undefined {
if (staleRecovery) return null;

Expand All @@ -276,12 +277,24 @@ export function backupSandboxStateForRebuild(
);
const hasAnyBackup = backup.backedUpDirs.length > 0 || backup.backedUpFiles.length > 0;
if (!backup.success && !hasAnyBackup) {
if (options?.force) {
console.warn(
` ${YW}⚠${R} Backup failed but --force was specified — skipping backup and rebuilding from registry metadata.`,
);
log(
"Force-skip: backup failed completely; continuing without backup as requested by --force",
);
return null;
}
console.error(" Failed to back up sandbox state.");
if (backup.error) console.error(` Reason: ${backup.error}`);
if (backup.failedDirs.length > 0) console.error(` Failed: ${backup.failedDirs.join(", ")}`);
if (backup.failedFiles.length > 0)
console.error(` Failed files: ${backup.failedFiles.join(", ")}`);
console.error(" Aborting rebuild to prevent data loss.");
console.error(
` Hint: use '${CLI_NAME} ${sandboxName} rebuild --force' to skip backup and rebuild from registry metadata.`,
);
relockShieldsIfNeeded(true);
bail("Failed to back up sandbox state.");
return undefined;
Expand Down
4 changes: 4 additions & 0 deletions src/lib/actions/sandbox/rebuild-pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

import type { RebuildSandboxOptions } from "../../domain/lifecycle/options";
import { normalizeRebuildSandboxOptions } from "../../domain/lifecycle/options";
import { BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV } from "../../inference/web-search";
import { MESSAGING_SETUP_APPLIER_ENV_KEY } from "../../messaging/applier/types";
import { MESSAGING_CHANNEL_CONFIG_ENV_KEYS } from "../../messaging-channel-config";
Expand Down Expand Up @@ -76,6 +77,7 @@ async function rebuildSandboxUnlocked(
options: string[] | RebuildSandboxOptions,
opts: RebuildSandboxExecutionOptions,
): Promise<void> {
const normalized = normalizeRebuildSandboxOptions(options);
const preflight = await runRebuildPreflightPhase(sandboxName, options, opts);
if (!preflight) return;
const {
Expand Down Expand Up @@ -161,6 +163,7 @@ async function rebuildSandboxUnlocked(
preparedRecoveryManifest: recoveryManifest,
messagingPlan,
webSearchConfig: durableConfig.webSearchConfig,
force: normalized.force,
log,
bail,
relockShieldsIfNeeded,
Expand Down Expand Up @@ -309,6 +312,7 @@ async function rebuildSandboxUnlocked(
backupManifest: backup.backupManifest,
mcpEntries: mcpPreparation.entries,
restoreSucceeded: restored.restoreSucceeded,
backupWasForceSkipped: backup.backupWasForceSkipped,
failedPresets: restored.failedPresets,
finalBuiltinPresets: restored.finalBuiltinPresets,
failedPresetRemovals: restored.failedPresetRemovals,
Expand Down
47 changes: 38 additions & 9 deletions src/lib/actions/sandbox/rebuild-post-restore-phase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export interface RebuildPostRestorePhaseInput {
backupManifest: RebuildBackupManifest;
mcpEntries: McpRebuildPreparation["entries"];
restoreSucceeded: boolean;
backupWasForceSkipped: boolean;
failedPresets: string[];
finalBuiltinPresets: string[];
failedPresetRemovals: string[];
Expand All @@ -45,6 +46,34 @@ export interface RebuildPostRestorePhaseInput {
bail: RebuildBail;
}

interface SuccessfulRebuildSummaryInput {
sandboxName: string;
backupManifest: RebuildBackupManifest;
backupWasForceSkipped: boolean;
staleRecovery: boolean;
rebuiltAgentName: string;
expectedVersion: string | null;
}

export function printSuccessfulRebuildSummary(
input: SuccessfulRebuildSummaryInput,
writeLine: (message: string) => void = console.log,
): void {
writeLine(` ${G}\u2713${R} Sandbox '${input.sandboxName}' rebuilt successfully`);
if (input.backupWasForceSkipped) {
writeLine(
` ${YW}\u26a0${R} Backup was skipped via --force after a total backup failure \u2014 prior workspace state was not preserved.`,
);
} else if (input.staleRecovery && !input.backupManifest) {
writeLine(
` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`,
);
}
if (input.expectedVersion) {
writeLine(` Now running: ${input.rebuiltAgentName} v${input.expectedVersion}`);
}
}

export function resolveRestoredPolicyRegistryState(
sandboxEntry: Pick<RebuildSandboxEntry, "policyPresetsFinalized">,
restoredBuiltinPresets: readonly string[],
Expand Down Expand Up @@ -77,6 +106,7 @@ export async function runRebuildPostRestorePhase(
backupManifest,
mcpEntries,
restoreSucceeded,
backupWasForceSkipped,
failedPresets,
finalBuiltinPresets,
failedPresetRemovals,
Expand Down Expand Up @@ -190,15 +220,14 @@ export async function runRebuildPostRestorePhase(
restoreSucceeded,
});
if (postRestoreComplete) {
console.log(` ${G}\u2713${R} Sandbox '${sandboxName}' rebuilt successfully`);
if (staleRecovery && !backupManifest) {
console.log(
` ${D}Recovered from a stale registry entry \u2014 no prior workspace state was available to restore.${R}`,
);
}
if (versionCheck.expectedVersion) {
console.log(` Now running: ${rebuiltAgentName} v${versionCheck.expectedVersion}`);
}
printSuccessfulRebuildSummary({
sandboxName,
backupManifest,
backupWasForceSkipped,
staleRecovery,
rebuiltAgentName,
expectedVersion: versionCheck.expectedVersion,
});
} else {
console.log(
` ${YW}\u26a0${R} Sandbox '${sandboxName}' rebuilt but some post-restore steps were incomplete`,
Expand Down
26 changes: 25 additions & 1 deletion src/lib/actions/sandbox/rebuild-restore-phase.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import * as policies from "../../policy";
import * as sandboxState from "../../state/sandbox";
import { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts";
import { resolveRestoredPolicyRegistryState } from "./rebuild-post-restore-phase";
import {
printSuccessfulRebuildSummary,
resolveRestoredPolicyRegistryState,
} from "./rebuild-post-restore-phase";
import { runRebuildRestorePhase } from "./rebuild-restore-phase";

const BUILTIN_OBSERVABILITY_CONTENT =
Expand Down Expand Up @@ -471,4 +474,25 @@ describe("rebuild policy restore fidelity", () => {
.policyPresetsFinalized,
).toBeUndefined();
});

it("retains the force-skipped backup warning in the successful final summary", () => {
const writeLine = vi.fn();

printSuccessfulRebuildSummary(
{
sandboxName: "alpha",
backupManifest: null,
backupWasForceSkipped: true,
staleRecovery: false,
rebuiltAgentName: "OpenClaw",
expectedVersion: "2026.6.10",
},
writeLine,
);

const output = writeLine.mock.calls.flat().join("\n");
expect(output).toContain("Sandbox 'alpha' rebuilt successfully");
expect(output).toContain("Backup was skipped via --force after a total backup failure");
expect(output).toContain("prior workspace state was not preserved");
});
});