[awf] cli-proxy: fail fast on external DIFC proxy unreachability

containers/cli-proxy/entrypoint.sh

@@ -56,6 +56,33 @@ export GIT_SSL_CAINFO="${COMBINED_CA}"
 
 echo "[cli-proxy] gh CLI configured to route through DIFC proxy at ${GH_HOST}"
 
+# Probe external DIFC proxy liveness before serving agent traffic.
+# If this fails, keep retries tightly bounded and fail startup early so
+# workflows do not enter long in-agent retry loops for connection-refused errors.
+MAX_LIVENESS_ATTEMPTS="${AWF_CLI_PROXY_LIVENESS_ATTEMPTS:-2}"
+LIVENESS_SLEEP_SECONDS="${AWF_CLI_PROXY_LIVENESS_SLEEP_SECONDS:-1}"
+LIVENESS_TIMEOUT_SECONDS="${AWF_CLI_PROXY_LIVENESS_TIMEOUT_SECONDS:-5}"
+ATTEMPT=1
+while [ "$ATTEMPT" -le "$MAX_LIVENESS_ATTEMPTS" ]; do
+  PROBE_ERR=""
+  if PROBE_ERR="$(timeout "${LIVENESS_TIMEOUT_SECONDS}" gh api rate_limit 2>&1 >/dev/null)"; then
+    echo "[cli-proxy] DIFC proxy liveness probe succeeded on attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}"
+    break
+  fi
+  PROBE_EXIT=$?
+  if [ "$ATTEMPT" -ge "$MAX_LIVENESS_ATTEMPTS" ]; then
+    echo "[cli-proxy] ERROR: DIFC proxy liveness probe failed for ${GH_HOST} (gh api exit=${PROBE_EXIT})"
+    if [ -n "${PROBE_ERR}" ]; then
+      echo "[cli-proxy] gh api error: ${PROBE_ERR}"
+    fi
+    echo "[cli-proxy] Failing fast to avoid repeated in-agent retries"
+    exit 1
+  fi
+  echo "[cli-proxy] DIFC proxy probe failed (attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}), retrying in ${LIVENESS_SLEEP_SECONDS}s..."
+  sleep "${LIVENESS_SLEEP_SECONDS}"
+  ATTEMPT=$((ATTEMPT + 1))
+done
+
 # Cleanup handler: stop the Node HTTP server and TCP tunnel on signal
 cleanup() {
   echo "[cli-proxy] Shutting down..."

src/container-lifecycle.ts

@@ -207,6 +207,12 @@ export async function startContainers(workDir: string, allowedDomains: string[],
     // don't fire two inspect calls when api-proxy is the root cause.
     const firstAttemptSquidStartupFailure = !firstAttemptApiProxyStartupFailure
       && await didContainerFailStartup(firstErrorMsg, SQUID_CONTAINER_NAME);
+    // CLI proxy startup failures are non-retriable because they usually mean
+    // the external DIFC proxy is unavailable (connection refused) and retries
+    // only delay failure while the agent repeatedly burns tokens.
+    const firstAttemptCliProxyStartupFailure = !firstAttemptApiProxyStartupFailure
+      && !firstAttemptSquidStartupFailure
+      && await didContainerFailStartup(firstErrorMsg, CLI_PROXY_CONTAINER_NAME);
 
     // When api-proxy or squid specifically fails to start, retry once.
     // Both containers are occasionally flaky on slow or busy CI runners:
@@ -246,12 +252,31 @@ export async function startContainers(workDir: string, allowedDomains: string[],
         if (await didContainerFailStartup(retryErrorMsg, SQUID_CONTAINER_NAME)) {
           await logContainerLogsToStderr(SQUID_CONTAINER_NAME);
         }
+        if (await didContainerFailStartup(retryErrorMsg, CLI_PROXY_CONTAINER_NAME)) {
+          await logContainerLogsToStderr(CLI_PROXY_CONTAINER_NAME);
+          throw new Error(
+            `AWF firewall failed to start: ${CLI_PROXY_CONTAINER_NAME} could not connect to the external DIFC proxy (or exited before establishing a connection). ` +
+            `Failing fast to avoid repeated in-agent retries. ` +
+            `The agent was never invoked. ` +
+            `See ${CLI_PROXY_CONTAINER_NAME} container logs above for details.`
+          );
+        }
         // Any remaining retry error (e.g. squid healthcheck or domain blockage) falls
         // through to the Squid log diagnostic path below as if it were the first error.
         return await handleHealthcheckError(retryErrorMsg, retryError as Error, workDir, proxyLogsDir, allowedDomains);
       }
     }
 
+    if (firstAttemptCliProxyStartupFailure) {
+      await logContainerLogsToStderr(CLI_PROXY_CONTAINER_NAME);
+      throw new Error(
+        `AWF firewall failed to start: ${CLI_PROXY_CONTAINER_NAME} could not connect to the external DIFC proxy (or exited before establishing a connection). ` +
+        `Failing fast to avoid repeated in-agent retries. ` +
+        `The agent was never invoked. ` +
+        `See ${CLI_PROXY_CONTAINER_NAME} container logs above for details.`
+      );
+    }
+
     return await handleHealthcheckError(firstErrorMsg, firstError as Error, workDir, proxyLogsDir, allowedDomains);
   }
 }

src/docker-manager-lifecycle.test.ts

@@ -311,6 +311,29 @@ describe('docker-manager lifecycle', () => {
       expect(squidInspectCalls).toHaveLength(0);
     });
 
+    it('fails fast when awf-cli-proxy startup fails and does not retry compose up', async () => {
+      // 1. docker rm (initial cleanup)
+      mockExecaFn.mockResolvedValueOnce({ stdout: '', stderr: '', exitCode: 0 } as any);
+      // 2. docker compose up (fails with cli-proxy unhealthy)
+      mockExecaFn.mockRejectedValueOnce(new Error('dependency failed to start: container awf-cli-proxy is unhealthy'));
+      // 3. docker inspect awf-api-proxy (fallback check - healthy)
+      mockExecaFn.mockResolvedValueOnce({ stdout: 'running|healthy', stderr: '', exitCode: 0 } as any);
+      // 4. docker inspect awf-squid (fallback check - healthy)
+      mockExecaFn.mockResolvedValueOnce({ stdout: 'running|healthy', stderr: '', exitCode: 0 } as any);
+      // 5. docker logs --tail 50 awf-cli-proxy (diagnostics before fail-fast throw)
+      mockExecaFn.mockResolvedValueOnce({ stdout: 'cli-proxy startup logs', stderr: '', exitCode: 0 } as any);
+
+      await expect(startContainers(testDir, ['github.com'])).rejects.toThrow(
+        'AWF firewall failed to start: awf-cli-proxy could not connect to the external DIFC proxy'
+      );
+
+      // Verify no retry happened: compose up should be called once
+      const upCalls = mockExecaFn.mock.calls.filter((call: any[]) =>
+        call[0] === 'docker' && Array.isArray(call[1]) && call[1].includes('up')
+      );
+      expect(upCalls).toHaveLength(1);
+    });
+
     it('should route retry error through Squid diagnostics when retry fails with non-api-proxy error', async () => {
       // Create access.log with denied entries so Squid diagnostics fire
       const squidLogsDir = path.join(testDir, 'squid-logs');