Fix Cloud Hypervisor API proxy readiness race

src/cloud-hypervisor-runtime-backend.test.ts

@@ -252,7 +252,7 @@ describe('Cloud Hypervisor runtime backend', () => {
           NO_PROXY: expect.stringContaining('awmg-mcpg'),
           no_proxy: expect.stringContaining('172.30.0.60'),
         }),
-        timeoutMs: 150_000,
+        timeoutMs: 210_000,
       }),
     );
   });
@@ -521,10 +521,15 @@ describe('Cloud Hypervisor runtime backend', () => {
     // The API proxy request must bypass the guest's HTTP(S)_PROXY env vars
     // (it targets the sidecar directly, not through Squid) and must only
     // run if the Squid reachability check already succeeded.
-    expect(script).toMatch(/nc -v -z .* && \(unset .*HTTP_PROXY.*; wget /);
+    expect(script).toMatch(/nc -v -z .* && \(unset .*HTTP_PROXY.*; attempt=1;/);
+    expect(script).toContain('while ! wget -q -T 20');
+    expect(script).toContain('if [ "$attempt" -ge 3 ]');
+    expect(script).toContain('API proxy /reflect unavailable after $attempt attempts');
+    expect(script).toContain('sleep "$delay"');
+    expect(script).toContain('delay=$((delay * 2))');
   });
 
-  it('uses generous nc/wget timeouts and an overall probe budget tolerant of nested-KVM scheduling delays', async () => {
+  it('uses bounded API proxy retries and an overall probe budget tolerant of nested-KVM scheduling delays', async () => {
     // Regression test: live-KVM validation confirmed (via captured host
     // network diagnostics) that the tap/nftables/vnet_hdr path was fully
     // correct -- Squid's response packets reached the host-side veth --
@@ -554,7 +559,9 @@ describe('Cloud Hypervisor runtime backend', () => {
     const script = probeCall.argv[2] as string;
     expect(script).toContain('nc -v -z -w 60');
     expect(script).toContain('wget -q -T 20');
-    expect(probeCall.timeoutMs).toBe(90_000);
+    expect(script).toContain('attempt=1; delay=2');
+    expect(script).toContain('if [ "$attempt" -ge 3 ]');
+    expect(probeCall.timeoutMs).toBe(150_000);
   });
 
   it('passes a beforeCleanup diagnostics hook to stop() on a startup failure, when --diagnostic-logs is set', async () => {

src/cloud-hypervisor-runtime-backend.ts

@@ -52,6 +52,11 @@ const CLOUD_HYPERVISOR_GUEST_HOME = `${CLOUD_HYPERVISOR_GUEST_WORKSPACE}/.awf-ho
  */
 const CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS = 90_000;
 const CLOUD_HYPERVISOR_GUEST_NETWORK_READY_TIMEOUT_MS = CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS;
+const CLOUD_HYPERVISOR_API_PROXY_PROBE_ATTEMPTS = 3;
+const CLOUD_HYPERVISOR_API_PROXY_PROBE_INITIAL_DELAY_SECONDS = 2;
+// Covers the 60-second Squid probe, three 20-second API proxy attempts,
+// their bounded backoff, and nested-KVM scheduling overhead.
+const CLOUD_HYPERVISOR_CONNECTIVITY_PROBE_TIMEOUT_MS = 150_000;
 const CLOUD_HYPERVISOR_CANCEL_GRACE_MS = 3_000;
 const CLOUD_HYPERVISOR_MAX_TIMEOUT_MS = 86_400_000;
 const MCP_GATEWAY_PORT = 8080;
@@ -472,7 +477,11 @@ export class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBacken
     const squidProbe = `nc -v -z -w 60 ${SQUID_IP} 3128`;
     const apiProxyProbe = this.config.enableApiProxy
       ? ` && (unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy; ` +
-        `wget -q -T 20 -O /dev/null http://${API_PROXY_IP}:10000/reflect)`
+        `attempt=1; delay=${CLOUD_HYPERVISOR_API_PROXY_PROBE_INITIAL_DELAY_SECONDS}; ` +
+        `while ! wget -q -T 20 -O /dev/null http://${API_PROXY_IP}:10000/reflect; do ` +
+        `if [ "$attempt" -ge ${CLOUD_HYPERVISOR_API_PROXY_PROBE_ATTEMPTS} ]; then ` +
+        `echo "API proxy /reflect unavailable after $attempt attempts" >&2; exit 1; fi; ` +
+        `sleep "$delay"; attempt=$((attempt + 1)); delay=$((delay * 2)); done)`
       : '';
     const topologyPeerProbe = Object.values(this.infrastructure?.topologyPeerIps ?? {})
       .map((ip) => ` && nc -v -z -w 60 ${ip} ${MCP_GATEWAY_PORT}`)
@@ -490,7 +499,8 @@ export class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBacken
       env: environment,
       cwd: CLOUD_HYPERVISOR_GUEST_WORKSPACE,
       ...identity,
-      timeoutMs: CLOUD_HYPERVISOR_PROBE_TIMEOUT_MS + topologyPeerCount * 60_000,
+      timeoutMs: CLOUD_HYPERVISOR_CONNECTIVITY_PROBE_TIMEOUT_MS +
+        topologyPeerCount * 60_000,
       stdout: stdoutCollector.stream,
       stderr: stderrCollector.stream,
     });