Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions src/lib/onboard/gateway-sandbox-reachability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,131 @@ describe("tryAutoApplyUfwRule (#4265)", () => {
});
});

describe("verifySandboxBridgeGatewayReachableOrExit host-gateway retry", () => {
const hostGatewayTcpFailure = {
ok: false as const,
reason: "tcp_failed" as const,
routeKind: "host_gateway" as const,
networkName: "openshell-docker",
gatewayIp: "192.168.65.254",
};

it("retries transient host-gateway tcp failures and returns when a later probe succeeds", async () => {
const reachabilityImpl = vi
.fn()
.mockResolvedValueOnce(hostGatewayTcpFailure)
.mockResolvedValueOnce({
...hostGatewayTcpFailure,
ok: true as const,
reason: "ok" as const,
});
const sleepMsImpl = vi.fn().mockResolvedValue(undefined);
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
try {
await verifySandboxBridgeGatewayReachableOrExit(true, {
reachabilityImpl,
retryAttempts: 3,
retryDelayMs: 25,
sleepMsImpl,
});
expect(reachabilityImpl).toHaveBeenCalledTimes(2);
expect(sleepMsImpl).toHaveBeenCalledTimes(1);
expect(sleepMsImpl).toHaveBeenCalledWith(25);
expect(log).toHaveBeenCalledWith(
expect.stringContaining("probe attempt 1/3 failed (tcp_failed)"),
);
expect(log).toHaveBeenCalledWith(expect.stringContaining("reachable on attempt 2/3"));
} finally {
log.mockRestore();
}
});

it("fails after exhausting persistent host-gateway tcp failures", async () => {
const reachabilityImpl = vi.fn().mockResolvedValue(hostGatewayTcpFailure);
const sleepMsImpl = vi.fn().mockResolvedValue(undefined);
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
try {
await expect(
verifySandboxBridgeGatewayReachableOrExit(false, {
reachabilityImpl,
retryAttempts: 3,
retryDelayMs: 25,
sleepMsImpl,
}),
).rejects.toThrow("sandbox-bridge unreachable");
expect(reachabilityImpl).toHaveBeenCalledTimes(3);
expect(sleepMsImpl).toHaveBeenCalledTimes(2);
expect(sleepMsImpl).toHaveBeenNthCalledWith(1, 25);
expect(sleepMsImpl).toHaveBeenNthCalledWith(2, 25);
expect(log).toHaveBeenCalledWith(
expect.stringContaining("probe attempt 1/3 failed (tcp_failed)"),
);
expect(log).toHaveBeenCalledWith(
expect.stringContaining("probe attempt 2/3 failed (tcp_failed)"),
);
const message = error.mock.calls[0]?.[0] as string;
expect(message).toContain("host-gateway route");
expect(message).not.toContain("ufw allow");
} finally {
log.mockRestore();
error.mockRestore();
}
});

it("uses the bounded production retry budget when retry options are not overridden", async () => {
const reachabilityImpl = vi.fn().mockResolvedValue(hostGatewayTcpFailure);
const sleepMsImpl = vi.fn().mockResolvedValue(undefined);
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
try {
await expect(
verifySandboxBridgeGatewayReachableOrExit(false, {
reachabilityImpl,
sleepMsImpl,
}),
).rejects.toThrow("sandbox-bridge unreachable");
expect(reachabilityImpl).toHaveBeenCalledTimes(10);
expect(sleepMsImpl).toHaveBeenCalledTimes(9);
expect(sleepMsImpl).toHaveBeenCalledWith(1000);
expect(log).toHaveBeenCalledWith(
expect.stringContaining("probe attempt 9/10 failed (tcp_failed)"),
);
} finally {
log.mockRestore();
error.mockRestore();
}
});

it("does not retry bridge-gateway tcp failures so UFW remediation remains responsible", async () => {
const bridgeGatewayTcpFailure = {
...hostGatewayTcpFailure,
routeKind: "bridge_gateway" as const,
subnet: "172.18.0.0/16",
gatewayIp: "172.18.0.1",
};
const reachabilityImpl = vi.fn().mockResolvedValue(bridgeGatewayTcpFailure);
const sleepMsImpl = vi.fn().mockResolvedValue(undefined);
const error = vi.spyOn(console, "error").mockImplementation(() => undefined);
try {
await expect(
verifySandboxBridgeGatewayReachableOrExit(false, {
autoApplyOptedInImpl: () => false,
reachabilityImpl,
retryAttempts: 3,
retryDelayMs: 25,
sleepMsImpl,
}),
).rejects.toThrow("sandbox-bridge unreachable");
expect(reachabilityImpl).toHaveBeenCalledTimes(1);
expect(sleepMsImpl).not.toHaveBeenCalled();
expect(error).toHaveBeenCalledWith(expect.stringContaining("ufw allow"));
} finally {
error.mockRestore();
}
});
});

describe("verifySandboxBridgeGatewayReachableOrExit UFW auto-apply (#4265)", () => {
const tcpFailure = {
ok: false as const,
Expand Down
33 changes: 33 additions & 0 deletions src/lib/onboard/gateway-sandbox-reachability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ const HOST_INTERNAL_NAME = "host.openshell.internal";
const HOST_DOCKER_INTERNAL_NAME = "host.docker.internal";
const DEFAULT_PROBE_TIMEOUT_SEC = 5;
const PROBE_RUN_OVERHEAD_MS = 10_000;
const DEFAULT_HOST_GATEWAY_RETRY_ATTEMPTS = 10;
const DEFAULT_HOST_GATEWAY_RETRY_DELAY_MS = 1000;

export type SandboxBridgeReachabilityReason =
| "ok"
Expand Down Expand Up @@ -484,6 +486,9 @@ interface SandboxBridgeVerifierOptions {
reach: SandboxBridgeReachabilityResult,
) => Promise<UfwAutoApplyResult> | UfwAutoApplyResult;
autoApplyOptedInImpl?: () => boolean;
retryAttempts?: number;
retryDelayMs?: number;
sleepMsImpl?: (ms: number) => Promise<void>;
}

const SILENT_UFW_AUTO_APPLY_REASONS = new Set<UfwAutoApplyResult["reason"]>([
Expand All @@ -492,6 +497,14 @@ const SILENT_UFW_AUTO_APPLY_REASONS = new Set<UfwAutoApplyResult["reason"]>([
"ufw_inactive",
]);

function isRetriableHostGatewayFailure(reach: SandboxBridgeReachabilityResult): boolean {
return reach.routeKind === "host_gateway" && reach.reason === "tcp_failed";
}

function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
}

export async function verifySandboxBridgeGatewayReachableOrExit(
exitOnFailure: boolean,
options: SandboxBridgeVerifierOptions = {},
Expand All @@ -512,6 +525,26 @@ export async function verifySandboxBridgeGatewayReachableOrExit(

let reach = await reachability();
if (reach.ok) return;
const retryAttempts = options.retryAttempts ?? DEFAULT_HOST_GATEWAY_RETRY_ATTEMPTS;
const retryDelayMs = options.retryDelayMs ?? DEFAULT_HOST_GATEWAY_RETRY_DELAY_MS;
const sleep = options.sleepMsImpl ?? sleepMs;
for (
let attempt = 2;
attempt <= retryAttempts && isRetriableHostGatewayFailure(reach);
attempt += 1
) {
console.log(
` Docker-driver sandbox bridge probe attempt ${attempt - 1}/${retryAttempts} failed (${reach.reason}); retrying in ${retryDelayMs} ms...`,
);
await sleep(retryDelayMs);
reach = await reachability();
if (reach.ok) {
console.log(
` ✓ Docker-driver sandbox bridge reachable on attempt ${attempt}/${retryAttempts}`,
);
return;
}
}

// #4265: when operator opts in and the probe proved a bridge TCP failure,
// try to auto-apply the firewall rule and re-probe before surfacing the
Expand Down