diff --git a/src/cloud-hypervisor-runtime-backend.test.ts b/src/cloud-hypervisor-runtime-backend.test.ts index 82be0cd62..9ccfdadd0 100644 --- a/src/cloud-hypervisor-runtime-backend.test.ts +++ b/src/cloud-hypervisor-runtime-backend.test.ts @@ -252,7 +252,7 @@ describe('Cloud Hypervisor runtime backend', () => { NO_PROXY: expect.stringContaining('awmg-mcpg'), no_proxy: expect.stringContaining('172.30.0.60'), }), - timeoutMs: 150_000, + timeoutMs: 210_000, }), ); }); @@ -521,10 +521,15 @@ describe('Cloud Hypervisor runtime backend', () => { // The API proxy request must bypass the guest's HTTP(S)_PROXY env vars // (it targets the sidecar directly, not through Squid) and must only // run if the Squid reachability check already succeeded. - expect(script).toMatch(/nc -v -z .* && \(unset .*HTTP_PROXY.*; wget /); + expect(script).toMatch(/nc -v -z .* && \(unset .*HTTP_PROXY.*; attempt=1;/); + expect(script).toContain('while ! wget -q -T 20'); + expect(script).toContain('if [ "$attempt" -ge 3 ]'); + expect(script).toContain('API proxy /reflect unavailable after $attempt attempts'); + expect(script).toContain('sleep "$delay"'); + expect(script).toContain('delay=$((delay * 2))'); }); - it('uses generous nc/wget timeouts and an overall probe budget tolerant of nested-KVM scheduling delays', async () => { + it('uses bounded API proxy retries and an overall probe budget tolerant of nested-KVM scheduling delays', async () => { // Regression test: live-KVM validation confirmed (via captured host // network diagnostics) that the tap/nftables/vnet_hdr path was fully // correct -- Squid's response packets reached the host-side veth -- @@ -554,7 +559,9 @@ describe('Cloud Hypervisor runtime backend', () => { const script = probeCall.argv[2] as string; expect(script).toContain('nc -v -z -w 60'); expect(script).toContain('wget -q -T 20'); - expect(probeCall.timeoutMs).toBe(90_000); + expect(script).toContain('attempt=1; delay=2'); + expect(script).toContain('if [ "$attempt" -ge 3 ]'); + expect(probeCall.timeoutMs).toBe(150_000); }); it('passes a beforeCleanup diagnostics hook to stop() on a startup failure, when --diagnostic-logs is set', async () => { diff --git a/src/cloud-hypervisor-runtime-backend.ts b/src/cloud-hypervisor-runtime-backend.ts index a1da56d30..d974c22c8 100644 --- a/src/cloud-hypervisor-runtime-backend.ts +++ b/src/cloud-hypervisor-runtime-backend.ts @@ -52,6 +52,11 @@ const CLOUD_HYPERVISOR_GUEST_HOME = `${CLOUD_HYPERVISOR_GUEST_WORKSPACE}/.awf-ho */ const CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS = 90_000; const CLOUD_HYPERVISOR_GUEST_NETWORK_READY_TIMEOUT_MS = CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS; +const CLOUD_HYPERVISOR_API_PROXY_PROBE_ATTEMPTS = 3; +const CLOUD_HYPERVISOR_API_PROXY_PROBE_INITIAL_DELAY_SECONDS = 2; +// Covers the 60-second Squid probe, three 20-second API proxy attempts, +// their bounded backoff, and nested-KVM scheduling overhead. +const CLOUD_HYPERVISOR_CONNECTIVITY_PROBE_TIMEOUT_MS = 150_000; const CLOUD_HYPERVISOR_CANCEL_GRACE_MS = 3_000; const CLOUD_HYPERVISOR_MAX_TIMEOUT_MS = 86_400_000; const MCP_GATEWAY_PORT = 8080; @@ -472,7 +477,11 @@ export class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBacken const squidProbe = `nc -v -z -w 60 ${SQUID_IP} 3128`; const apiProxyProbe = this.config.enableApiProxy ? ` && (unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy; ` + - `wget -q -T 20 -O /dev/null http://${API_PROXY_IP}:10000/reflect)` + `attempt=1; delay=${CLOUD_HYPERVISOR_API_PROXY_PROBE_INITIAL_DELAY_SECONDS}; ` + + `while ! wget -q -T 20 -O /dev/null http://${API_PROXY_IP}:10000/reflect; do ` + + `if [ "$attempt" -ge ${CLOUD_HYPERVISOR_API_PROXY_PROBE_ATTEMPTS} ]; then ` + + `echo "API proxy /reflect unavailable after $attempt attempts" >&2; exit 1; fi; ` + + `sleep "$delay"; attempt=$((attempt + 1)); delay=$((delay * 2)); done)` : ''; const topologyPeerProbe = Object.values(this.infrastructure?.topologyPeerIps ?? {}) .map((ip) => ` && nc -v -z -w 60 ${ip} ${MCP_GATEWAY_PORT}`) @@ -490,7 +499,8 @@ export class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBacken env: environment, cwd: CLOUD_HYPERVISOR_GUEST_WORKSPACE, ...identity, - timeoutMs: CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS + topologyPeerCount * 60_000, + timeoutMs: CLOUD_HYPERVISOR_CONNECTIVITY_PROBE_TIMEOUT_MS + + topologyPeerCount * 60_000, stdout: stdoutCollector.stream, stderr: stderrCollector.stream, });