diff --git a/.github/workflows/red-team-benchmark.lock.yml b/.github/workflows/red-team-benchmark.lock.yml index 20ab49f0c..425e006c6 100644 --- a/.github/workflows/red-team-benchmark.lock.yml +++ b/.github/workflows/red-team-benchmark.lock.yml @@ -397,7 +397,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} id: awf_run name: Run AWF-protected benchmark (victim inside AWF sandbox) - run: "mkdir -p /tmp/gh-aw/agent/awf\nmkdir -p /tmp/gh-aw/agent/awf/firewall-logs\nAWF_LEAKS=\"n/a\"\nAWF_BLOCKED=\"n/a\"\nif [ -z \"$ANTHROPIC_API_KEY\" ] || [ -z \"$OPENAI_API_KEY\" ]; then\n echo \"::warning::Missing API keys — AWF-protected run skipped\"\n echo '{\"skipped\":true,\"reason\":\"missing API keys\"}' > /tmp/gh-aw/agent/awf/summary.json\nelif ! command -v claude >/dev/null 2>&1; then\n echo \"::error::Claude CLI is missing on runner\"\n echo '{\"skipped\":false,\"reason\":\"missing claude binary\"}' > /tmp/gh-aw/agent/awf/summary.json\n exit 1\nelse\n cd /tmp/adversarial_dojo\n # Run the benchmark inside AWF sandbox — benchmark traffic is restricted\n # to api.anthropic.com and api.openai.com, blocking other egress attempts.\n sudo awf \\\n --allow-domains api.anthropic.com,api.openai.com \\\n --proxy-logs-dir /tmp/gh-aw/agent/awf/firewall-logs \\\n --log-level info \\\n -- \"$HOME/.local/bin/uv\" run adversarial-dojo search-attacks \\\n /tmp/awf-benchmark.toml \\\n --out /tmp/gh-aw/agent/awf \\\n 2>/tmp/gh-aw/agent/awf/stderr.log || true\n if [ -f /tmp/gh-aw/agent/awf/summary.json ]; then\n AWF_LEAKS=$(jq -r '.leak_events | length' /tmp/gh-aw/agent/awf/summary.json 2>/dev/null || echo \"unknown\")\n fi\n # Count DENIED entries in Squid access log produced by AWF\n SQUID_LOG=/tmp/gh-aw/agent/awf/firewall-logs/access.log\n if [ ! -f \"$SQUID_LOG\" ]; then\n SQUID_LOG=$(find /tmp -name 'access.log' -path '*awf*' 2>/dev/null | head -1)\n fi\n if [ -n \"$SQUID_LOG\" ]; then\n AWF_BLOCKED=$(grep -c \"DENIED\" \"$SQUID_LOG\" 2>/dev/null || true)\n cp \"$SQUID_LOG\" /tmp/gh-aw/agent/squid-access.log\n else\n echo \"No Squid access log found\" > /tmp/gh-aw/agent/squid-access.log\n AWF_BLOCKED=\"0\"\n fi\n echo \"AWF-protected — leaks: $AWF_LEAKS, blocked requests: $AWF_BLOCKED\"\nfi\necho \"AWF_LEAKS=$AWF_LEAKS\" >> \"$GITHUB_OUTPUT\"\necho \"AWF_BLOCKED=$AWF_BLOCKED\" >> \"$GITHUB_OUTPUT\"\n" + run: "mkdir -p /tmp/gh-aw/agent/awf\nmkdir -p /tmp/gh-aw/agent/awf/firewall-logs\nAWF_LEAKS=\"n/a\"\nAWF_BLOCKED=\"n/a\"\nif [ -z \"$ANTHROPIC_API_KEY\" ] || [ -z \"$OPENAI_API_KEY\" ]; then\n echo \"::warning::Missing API keys — AWF-protected run skipped\"\n echo '{\"skipped\":true,\"reason\":\"missing API keys\"}' > /tmp/gh-aw/agent/awf/summary.json\nelif ! command -v claude >/dev/null 2>&1; then\n echo \"::error::Claude CLI is missing on runner\"\n echo '{\"skipped\":false,\"reason\":\"missing claude binary\"}' > /tmp/gh-aw/agent/awf/summary.json\n exit 1\nelse\n # Run the benchmark inside AWF sandbox — benchmark traffic is restricted\n # to api.anthropic.com and api.openai.com, blocking other egress attempts.\n # Mount adversarial_dojo (with its uv-managed venv), the uv binary, config\n # files and the output directory so the benchmark tooling is available\n # inside the minimal AWF container image.\n sudo awf \\\n --allow-domains api.anthropic.com,api.openai.com \\\n --proxy-logs-dir /tmp/gh-aw/agent/awf/firewall-logs \\\n --log-level info \\\n --mount /tmp/adversarial_dojo:/tmp/adversarial_dojo \\\n --mount \"$HOME/.local/bin/uv:$HOME/.local/bin/uv:ro\" \\\n --mount /tmp/awf-benchmark.toml:/tmp/awf-benchmark.toml:ro \\\n --mount /tmp/awf-benchmark:/tmp/awf-benchmark:ro \\\n --mount /tmp/gh-aw/agent/awf:/tmp/gh-aw/agent/awf \\\n --container-workdir /tmp/adversarial_dojo \\\n --env \"ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY\" \\\n --env \"OPENAI_API_KEY=$OPENAI_API_KEY\" \\\n -- \"$HOME/.local/bin/uv\" run adversarial-dojo search-attacks \\\n /tmp/awf-benchmark.toml \\\n --out /tmp/gh-aw/agent/awf \\\n 2>/tmp/gh-aw/agent/awf/stderr.log || true\n if [ -f /tmp/gh-aw/agent/awf/summary.json ]; then\n AWF_LEAKS=$(jq -r '.leak_events | length' /tmp/gh-aw/agent/awf/summary.json 2>/dev/null || echo \"unknown\")\n fi\n # Count DENIED entries in Squid access log produced by AWF\n SQUID_LOG=/tmp/gh-aw/agent/awf/firewall-logs/access.log\n if [ ! -f \"$SQUID_LOG\" ]; then\n SQUID_LOG=$(find /tmp -name 'access.log' -path '*awf*' 2>/dev/null | head -1)\n fi\n if [ -n \"$SQUID_LOG\" ]; then\n AWF_BLOCKED=$(grep -c \"DENIED\" \"$SQUID_LOG\" 2>/dev/null || true)\n cp \"$SQUID_LOG\" /tmp/gh-aw/agent/squid-access.log\n else\n echo \"No Squid access log found\" > /tmp/gh-aw/agent/squid-access.log\n AWF_BLOCKED=\"0\"\n fi\n echo \"AWF-protected — leaks: $AWF_LEAKS, blocked requests: $AWF_BLOCKED\"\nfi\necho \"AWF_LEAKS=$AWF_LEAKS\" >> \"$GITHUB_OUTPUT\"\necho \"AWF_BLOCKED=$AWF_BLOCKED\" >> \"$GITHUB_OUTPUT\"\n" - env: EXPR_AWF_BLOCKED: ${{ steps.awf_run.outputs.AWF_BLOCKED }} EXPR_AWF_LEAKS: ${{ steps.awf_run.outputs.AWF_LEAKS }} diff --git a/.github/workflows/red-team-benchmark.md b/.github/workflows/red-team-benchmark.md index 4a9e48a58..c4cc28a02 100644 --- a/.github/workflows/red-team-benchmark.md +++ b/.github/workflows/red-team-benchmark.md @@ -176,13 +176,23 @@ steps: echo '{"skipped":false,"reason":"missing claude binary"}' > /tmp/gh-aw/agent/awf/summary.json exit 1 else - cd /tmp/adversarial_dojo # Run the benchmark inside AWF sandbox — benchmark traffic is restricted # to api.anthropic.com and api.openai.com, blocking other egress attempts. + # Mount adversarial_dojo (with its uv-managed venv), the uv binary, config + # files and the output directory so the benchmark tooling is available + # inside the minimal AWF container image. sudo awf \ --allow-domains api.anthropic.com,api.openai.com \ --proxy-logs-dir /tmp/gh-aw/agent/awf/firewall-logs \ --log-level info \ + --mount /tmp/adversarial_dojo:/tmp/adversarial_dojo \ + --mount "$HOME/.local/bin/uv:$HOME/.local/bin/uv:ro" \ + --mount /tmp/awf-benchmark.toml:/tmp/awf-benchmark.toml:ro \ + --mount /tmp/awf-benchmark:/tmp/awf-benchmark:ro \ + --mount /tmp/gh-aw/agent/awf:/tmp/gh-aw/agent/awf \ + --container-workdir /tmp/adversarial_dojo \ + --env "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" \ + --env "OPENAI_API_KEY=$OPENAI_API_KEY" \ -- "$HOME/.local/bin/uv" run adversarial-dojo search-attacks \ /tmp/awf-benchmark.toml \ --out /tmp/gh-aw/agent/awf \ diff --git a/scripts/ci/red-team-benchmark-workflow.test.ts b/scripts/ci/red-team-benchmark-workflow.test.ts index 21b81a6e9..d57e9cfc0 100644 --- a/scripts/ci/red-team-benchmark-workflow.test.ts +++ b/scripts/ci/red-team-benchmark-workflow.test.ts @@ -69,6 +69,16 @@ describe('red-team benchmark workflow config', () => { expect(source).toContain('--allow-domains api.anthropic.com,api.openai.com'); expect(source).toContain('--proxy-logs-dir /tmp/gh-aw/agent/awf/firewall-logs'); + // adversarial_dojo tooling is mounted into the AWF container + expect(source).toContain('--mount /tmp/adversarial_dojo:/tmp/adversarial_dojo'); + expect(source).toContain('--mount /tmp/awf-benchmark.toml:/tmp/awf-benchmark.toml:ro'); + expect(source).toContain('--mount /tmp/gh-aw/agent/awf:/tmp/gh-aw/agent/awf'); + expect(source).toContain('--container-workdir /tmp/adversarial_dojo'); + + // API keys are explicitly forwarded to the AWF container + expect(source).toContain('--env "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY"'); + expect(source).toContain('--env "OPENAI_API_KEY=$OPENAI_API_KEY"'); + // Both benchmark runs expect(source).toContain('Run baseline benchmark (victim without AWF)'); expect(source).toContain('Run AWF-protected benchmark (victim inside AWF sandbox)'); @@ -141,6 +151,10 @@ describe('red-team benchmark workflow config', () => { expect(lock).toContain('f51227612e43d98658679710d5505989e7f53ec7'); expect(lock).toContain('--out'); + // adversarial_dojo mounts compiled into lock + expect(lock).toContain('--mount /tmp/adversarial_dojo:/tmp/adversarial_dojo'); + expect(lock).toContain('--container-workdir /tmp/adversarial_dojo'); + // Benchmark steps present expect(lock).toContain('baseline'); expect(lock).toContain('awf_run');