diff --git a/containers/cli-proxy/entrypoint.sh b/containers/cli-proxy/entrypoint.sh index c6600cf04..96afd98fd 100644 --- a/containers/cli-proxy/entrypoint.sh +++ b/containers/cli-proxy/entrypoint.sh @@ -56,6 +56,33 @@ export GIT_SSL_CAINFO="${COMBINED_CA}" echo "[cli-proxy] gh CLI configured to route through DIFC proxy at ${GH_HOST}" +# Probe external DIFC proxy liveness before serving agent traffic. +# If this fails, keep retries tightly bounded and fail startup early so +# workflows do not enter long in-agent retry loops for connection-refused errors. +MAX_LIVENESS_ATTEMPTS="${AWF_CLI_PROXY_LIVENESS_ATTEMPTS:-2}" +LIVENESS_SLEEP_SECONDS="${AWF_CLI_PROXY_LIVENESS_SLEEP_SECONDS:-1}" +LIVENESS_TIMEOUT_SECONDS="${AWF_CLI_PROXY_LIVENESS_TIMEOUT_SECONDS:-5}" +ATTEMPT=1 +while [ "$ATTEMPT" -le "$MAX_LIVENESS_ATTEMPTS" ]; do + PROBE_ERR="" + if PROBE_ERR="$(timeout "${LIVENESS_TIMEOUT_SECONDS}" gh api rate_limit 2>&1 >/dev/null)"; then + echo "[cli-proxy] DIFC proxy liveness probe succeeded on attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}" + break + fi + PROBE_EXIT=$? + if [ "$ATTEMPT" -ge "$MAX_LIVENESS_ATTEMPTS" ]; then + echo "[cli-proxy] ERROR: DIFC proxy liveness probe failed for ${GH_HOST} (gh api exit=${PROBE_EXIT})" + if [ -n "${PROBE_ERR}" ]; then + echo "[cli-proxy] gh api error: ${PROBE_ERR}" + fi + echo "[cli-proxy] Failing fast to avoid repeated in-agent retries" + exit 1 + fi + echo "[cli-proxy] DIFC proxy probe failed (attempt ${ATTEMPT}/${MAX_LIVENESS_ATTEMPTS}), retrying in ${LIVENESS_SLEEP_SECONDS}s..." + sleep "${LIVENESS_SLEEP_SECONDS}" + ATTEMPT=$((ATTEMPT + 1)) +done + # Cleanup handler: stop the Node HTTP server and TCP tunnel on signal cleanup() { echo "[cli-proxy] Shutting down..." diff --git a/src/container-lifecycle.ts b/src/container-lifecycle.ts index eb8cb4e9d..11d4de131 100644 --- a/src/container-lifecycle.ts +++ b/src/container-lifecycle.ts @@ -207,6 +207,12 @@ export async function startContainers(workDir: string, allowedDomains: string[], // don't fire two inspect calls when api-proxy is the root cause. const firstAttemptSquidStartupFailure = !firstAttemptApiProxyStartupFailure && await didContainerFailStartup(firstErrorMsg, SQUID_CONTAINER_NAME); + // CLI proxy startup failures are non-retriable because they usually mean + // the external DIFC proxy is unavailable (connection refused) and retries + // only delay failure while the agent repeatedly burns tokens. + const firstAttemptCliProxyStartupFailure = !firstAttemptApiProxyStartupFailure + && !firstAttemptSquidStartupFailure + && await didContainerFailStartup(firstErrorMsg, CLI_PROXY_CONTAINER_NAME); // When api-proxy or squid specifically fails to start, retry once. // Both containers are occasionally flaky on slow or busy CI runners: @@ -246,12 +252,31 @@ export async function startContainers(workDir: string, allowedDomains: string[], if (await didContainerFailStartup(retryErrorMsg, SQUID_CONTAINER_NAME)) { await logContainerLogsToStderr(SQUID_CONTAINER_NAME); } + if (await didContainerFailStartup(retryErrorMsg, CLI_PROXY_CONTAINER_NAME)) { + await logContainerLogsToStderr(CLI_PROXY_CONTAINER_NAME); + throw new Error( + `AWF firewall failed to start: ${CLI_PROXY_CONTAINER_NAME} could not connect to the external DIFC proxy (or exited before establishing a connection). ` + + `Failing fast to avoid repeated in-agent retries. ` + + `The agent was never invoked. ` + + `See ${CLI_PROXY_CONTAINER_NAME} container logs above for details.` + ); + } // Any remaining retry error (e.g. squid healthcheck or domain blockage) falls // through to the Squid log diagnostic path below as if it were the first error. return await handleHealthcheckError(retryErrorMsg, retryError as Error, workDir, proxyLogsDir, allowedDomains); } } + if (firstAttemptCliProxyStartupFailure) { + await logContainerLogsToStderr(CLI_PROXY_CONTAINER_NAME); + throw new Error( + `AWF firewall failed to start: ${CLI_PROXY_CONTAINER_NAME} could not connect to the external DIFC proxy (or exited before establishing a connection). ` + + `Failing fast to avoid repeated in-agent retries. ` + + `The agent was never invoked. ` + + `See ${CLI_PROXY_CONTAINER_NAME} container logs above for details.` + ); + } + return await handleHealthcheckError(firstErrorMsg, firstError as Error, workDir, proxyLogsDir, allowedDomains); } } diff --git a/src/docker-manager-lifecycle.test.ts b/src/docker-manager-lifecycle.test.ts index ccb8fe628..02d4b850b 100644 --- a/src/docker-manager-lifecycle.test.ts +++ b/src/docker-manager-lifecycle.test.ts @@ -311,6 +311,29 @@ describe('docker-manager lifecycle', () => { expect(squidInspectCalls).toHaveLength(0); }); + it('fails fast when awf-cli-proxy startup fails and does not retry compose up', async () => { + // 1. docker rm (initial cleanup) + mockExecaFn.mockResolvedValueOnce({ stdout: '', stderr: '', exitCode: 0 } as any); + // 2. docker compose up (fails with cli-proxy unhealthy) + mockExecaFn.mockRejectedValueOnce(new Error('dependency failed to start: container awf-cli-proxy is unhealthy')); + // 3. docker inspect awf-api-proxy (fallback check - healthy) + mockExecaFn.mockResolvedValueOnce({ stdout: 'running|healthy', stderr: '', exitCode: 0 } as any); + // 4. docker inspect awf-squid (fallback check - healthy) + mockExecaFn.mockResolvedValueOnce({ stdout: 'running|healthy', stderr: '', exitCode: 0 } as any); + // 5. docker logs --tail 50 awf-cli-proxy (diagnostics before fail-fast throw) + mockExecaFn.mockResolvedValueOnce({ stdout: 'cli-proxy startup logs', stderr: '', exitCode: 0 } as any); + + await expect(startContainers(testDir, ['github.com'])).rejects.toThrow( + 'AWF firewall failed to start: awf-cli-proxy could not connect to the external DIFC proxy' + ); + + // Verify no retry happened: compose up should be called once + const upCalls = mockExecaFn.mock.calls.filter((call: any[]) => + call[0] === 'docker' && Array.isArray(call[1]) && call[1].includes('up') + ); + expect(upCalls).toHaveLength(1); + }); + it('should route retry error through Squid diagnostics when retry fails with non-api-proxy error', async () => { // Create access.log with denied entries so Squid diagnostics fire const squidLogsDir = path.join(testDir, 'squid-logs');