From 23245e6f7bfeb0cfb6bb31aa2319a918faf15929 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Mon, 23 Mar 2026 18:21:33 +0000 Subject: [PATCH 1/4] feat: add firewall audit/observability and policy logging Close auditing gaps flagged by ProdSec security review requiring all agentic network interactions be logged and auditable. Changes: - Add --audit-dir flag (+ AWF_AUDIT_DIR env var) for configurable audit artifact directory - Preserve squid.conf, redacted docker-compose.yml, and policy manifest as audit artifacts after each run - Generate policy-manifest.json describing all firewall rules with evaluation order, enabling deterministic "which rule matched?" analysis - Add structured JSONL audit log (audit.jsonl) alongside existing text access.log for machine-readable per-request logging - Add iptables LOG targets before DROP rules (rate-limited) and capture full iptables-save state for audit trail - Add rule matching enrichment that replays ACL evaluation to attribute each log entry to the specific policy rule that caused allow/deny - Add `awf logs audit` command with --rule, --domain, --decision filters - Enhance `awf logs stats/summary` with per-rule hit counts when manifest is available Addresses: github/agentic-workflows#174 Co-Authored-By: Claude Opus 4.6 (1M context) --- containers/agent/setup-iptables.sh | 29 +++ src/cli.ts | 42 +++- src/commands/logs-audit.ts | 190 +++++++++++++++ src/commands/logs-command-helpers.ts | 55 ++++- src/docker-manager.ts | 105 ++++++++- src/logs/audit-enricher.test.ts | 340 +++++++++++++++++++++++++++ src/logs/audit-enricher.ts | 166 +++++++++++++ src/logs/log-aggregator.ts | 71 ++++-- src/logs/log-parser.test.ts | 49 +++- src/logs/log-parser.ts | 54 +++++ src/logs/stats-formatter.ts | 35 ++- src/squid-config.test.ts | 144 +++++++++++- src/squid-config.ts | 266 ++++++++++++++++++--- src/types.ts | 74 ++++++ 14 files changed, 1558 insertions(+), 62 deletions(-) create mode 100644 src/commands/logs-audit.ts create mode 100644 src/logs/audit-enricher.test.ts create mode 100644 src/logs/audit-enricher.ts diff --git a/containers/agent/setup-iptables.sh b/containers/agent/setup-iptables.sh index 15247c28a..f54f517a8 100644 --- a/containers/agent/setup-iptables.sh +++ b/containers/agent/setup-iptables.sh @@ -312,11 +312,20 @@ if [ -n "$AWF_API_PROXY_IP" ]; then iptables -A OUTPUT -p tcp -d "$AWF_API_PROXY_IP" -j ACCEPT fi +# Log dangerous port access attempts for audit (rate-limited to avoid log flooding) +# These ports are blocked by NAT RETURN + final DROP, but logging helps identify +# what the agent tried to access +echo "[iptables] Adding audit LOG rules for dangerous ports and default deny..." +iptables -A OUTPUT -p tcp -m multiport --dports 22,23,25,110,143,445,1433,1521,3306,3389,5432,6379,27017,27018,28017 \ + -m limit --limit 5/min --limit-burst 10 -j LOG --log-prefix "[FW_BLOCKED_DANGEROUS_PORT] " --log-level 4 --log-uid + # Drop all other TCP and UDP traffic (default deny policy) # TCP: ensures only explicitly allowed ports can be accessed # UDP: prevents DNS exfiltration by blocking direct queries to non-configured DNS servers echo "[iptables] Drop all non-allowed TCP and UDP traffic (default deny)..." +iptables -A OUTPUT -p tcp -m limit --limit 10/min --limit-burst 20 -j LOG --log-prefix "[FW_BLOCKED_TCP] " --log-level 4 --log-uid iptables -A OUTPUT -p tcp -j DROP +iptables -A OUTPUT -p udp -m limit --limit 10/min --limit-burst 20 -j LOG --log-prefix "[FW_BLOCKED_UDP_AGENT] " --log-level 4 --log-uid iptables -A OUTPUT -p udp -j DROP echo "[iptables] NAT rules applied successfully" @@ -328,3 +337,23 @@ if [ "$IP6TABLES_AVAILABLE" = true ]; then else echo "[iptables] (ip6tables NAT not available)" fi + +# Dump full iptables state for audit trail +# Written to the init signal volume so it can be preserved by the host +AUDIT_FILE="/tmp/awf-init/iptables-audit.txt" +echo "# iptables audit dump - $(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "$AUDIT_FILE" +echo "" >> "$AUDIT_FILE" +echo "## IPv4 NAT rules" >> "$AUDIT_FILE" +iptables-save -t nat >> "$AUDIT_FILE" 2>/dev/null || echo "(iptables-save not available)" >> "$AUDIT_FILE" +echo "" >> "$AUDIT_FILE" +echo "## IPv4 filter rules" >> "$AUDIT_FILE" +iptables-save -t filter >> "$AUDIT_FILE" 2>/dev/null || echo "(iptables-save not available)" >> "$AUDIT_FILE" +if [ "$IP6TABLES_AVAILABLE" = true ]; then + echo "" >> "$AUDIT_FILE" + echo "## IPv6 NAT rules" >> "$AUDIT_FILE" + ip6tables-save -t nat >> "$AUDIT_FILE" 2>/dev/null || true + echo "" >> "$AUDIT_FILE" + echo "## IPv6 filter rules" >> "$AUDIT_FILE" + ip6tables-save -t filter >> "$AUDIT_FILE" 2>/dev/null || true +fi +echo "[iptables] Audit state dumped to $AUDIT_FILE" diff --git a/src/cli.ts b/src/cli.ts index fe5f5f0cc..9b038da77 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,7 @@ import { runAgentCommand, stopContainers, cleanup, + preserveIptablesAudit, } from './docker-manager'; import { ensureFirewallNetwork, @@ -1323,6 +1324,10 @@ program '--proxy-logs-dir ', 'Directory to save Squid proxy access.log' ) + .option( + '--audit-dir ', + 'Directory for firewall audit artifacts (configs, policy manifest, iptables state)' + ) .argument('[args...]', 'Command and arguments to execute (use -- to separate from options)') .action(async (args: string[], options) => { // Require -- separator for passing command arguments @@ -1619,6 +1624,7 @@ program dnsOverHttps, memoryLimit: memoryLimit.value, proxyLogsDir: options.proxyLogsDir, + auditDir: options.auditDir || process.env.AWF_AUDIT_DIR, enableHostAccess: options.enableHostAccess, allowHostPorts: options.allowHostPorts, sslBump: options.sslBump, @@ -1737,7 +1743,9 @@ program logger.info(`Received ${signal}, cleaning up...`); } + // Copy iptables audit BEFORE stopping containers (volumes are destroyed by `docker compose down -v`) if (containersStarted) { + preserveIptablesAudit(config.workDir, config.auditDir); await stopContainers(config.workDir, config.keepContainers); } @@ -1746,7 +1754,7 @@ program } if (!config.keepContainers) { - await cleanup(config.workDir, false, config.proxyLogsDir); + await cleanup(config.workDir, false, config.proxyLogsDir, config.auditDir); // Note: We don't remove the firewall network here since it can be reused // across multiple runs. Cleanup script will handle removal if needed. } else { @@ -1940,6 +1948,38 @@ logsCmd }); }); +// Logs audit subcommand - show enriched audit with rule matching +logsCmd + .command('audit') + .description('Show firewall audit with policy rule matching (requires policy-manifest.json)') + .option( + '--format ', + 'Output format: json, markdown, pretty', + 'pretty' + ) + .option('--source ', 'Path to log directory or "running" for live container') + .option('--rule ', 'Filter to specific rule ID') + .option('--domain ', 'Filter to specific domain') + .option('--decision ', 'Filter to "allowed" or "denied"') + .action(async (options) => { + const validFormats = ['json', 'markdown', 'pretty']; + validateFormat(options.format, validFormats); + + if (options.decision && !['allowed', 'denied'].includes(options.decision)) { + logger.error(`Invalid decision filter: ${options.decision}. Must be "allowed" or "denied".`); + process.exit(1); + } + + const { auditCommand } = await import('./commands/logs-audit'); + await auditCommand({ + format: options.format as 'json' | 'markdown' | 'pretty', + source: options.source, + rule: options.rule, + domain: options.domain, + decision: options.decision, + }); + }); + // Only parse arguments if this file is run directly (not imported as a module) if (require.main === module) { program.parse(); diff --git a/src/commands/logs-audit.ts b/src/commands/logs-audit.ts new file mode 100644 index 000000000..ad89423dd --- /dev/null +++ b/src/commands/logs-audit.ts @@ -0,0 +1,190 @@ +/** + * Command handler for `awf logs audit` subcommand + * + * Enriches firewall logs with policy rule matching when a policy-manifest.json + * is available alongside the log files. Shows which specific rule caused each + * allow/deny decision. + */ + +import chalk from 'chalk'; +import type { LogStatsFormat, PolicyManifest } from '../types'; +import { loadAllLogs } from '../logs/log-aggregator'; +import { enrichWithPolicyRules, computeRuleStats, EnrichedLogEntry } from '../logs/audit-enricher'; +import { + discoverAndSelectSource, + findPolicyManifestForSource, +} from './logs-command-helpers'; +import { logger } from '../logger'; + +export interface AuditCommandOptions { + format: LogStatsFormat; + source?: string; + /** Filter to specific rule ID */ + rule?: string; + /** Filter to specific domain */ + domain?: string; + /** Filter to 'allowed' or 'denied' */ + decision?: 'allowed' | 'denied'; +} + +function formatAuditJson(entries: EnrichedLogEntry[]): string { + return entries.map(e => JSON.stringify({ + timestamp: e.timestamp, + domain: e.domain, + method: e.method, + status: e.statusCode, + decision: e.isAllowed ? 'allowed' : 'denied', + matchedRule: e.matchedRuleId, + matchReason: e.matchReason, + url: e.url, + })).join('\n'); +} + +function formatAuditMarkdown(entries: EnrichedLogEntry[], manifest: PolicyManifest): string { + const lines: string[] = []; + const ruleStats = computeRuleStats(entries, manifest); + + lines.push('## Firewall Audit Report\n'); + + // Policy summary + lines.push('### Active Policy\n'); + lines.push(`- **SSL Bump**: ${manifest.sslBumpEnabled ? 'enabled' : 'disabled'}`); + lines.push(`- **DLP**: ${manifest.dlpEnabled ? 'enabled' : 'disabled'}`); + lines.push(`- **Host Access**: ${manifest.hostAccessEnabled ? 'enabled' : 'disabled'}`); + lines.push(`- **DNS Servers**: ${manifest.dnsServers.join(', ')}`); + lines.push(`- **Dangerous Ports Blocked**: ${manifest.dangerousPorts.length} ports\n`); + + // Rule hits table + lines.push('### Rule Evaluation\n'); + lines.push('| Rule | Action | Hits | Description |'); + lines.push('|------|--------|------|-------------|'); + for (const rule of ruleStats) { + const actionIcon = rule.action === 'allow' ? '✅' : '🚫'; + const hitsStr = rule.hits > 0 ? `**${rule.hits}**` : '0'; + lines.push(`| ${rule.ruleId} | ${actionIcon} ${rule.action} | ${hitsStr} | ${rule.description} |`); + } + + // Denied requests detail + const denied = entries.filter(e => !e.isAllowed && e.url !== 'error:transaction-end-before-headers'); + if (denied.length > 0) { + lines.push('\n### Denied Requests\n'); + lines.push('| Timestamp | Domain | Rule | Reason |'); + lines.push('|-----------|--------|------|--------|'); + for (const entry of denied.slice(0, 50)) { // Cap at 50 + const ts = new Date(entry.timestamp * 1000).toISOString(); + lines.push(`| ${ts} | ${entry.domain} | ${entry.matchedRuleId} | ${entry.matchReason} |`); + } + if (denied.length > 50) { + lines.push(`\n_...and ${denied.length - 50} more denied requests_`); + } + } + + return lines.join('\n'); +} + +function formatAuditPretty(entries: EnrichedLogEntry[], manifest: PolicyManifest, colorize: boolean): string { + const c = colorize + ? chalk + : (new Proxy({}, { get: () => (s: string) => s }) as typeof chalk); + + const lines: string[] = []; + const ruleStats = computeRuleStats(entries, manifest); + + lines.push(c.bold('Firewall Audit Report')); + lines.push(c.gray('─'.repeat(60))); + lines.push(''); + + // Rule hits + lines.push(c.bold('Rule Evaluation:')); + const maxIdLen = Math.max(...ruleStats.map(r => r.ruleId.length)); + for (const rule of ruleStats) { + const paddedId = rule.ruleId.padEnd(maxIdLen + 2); + const actionStr = rule.action === 'allow' ? c.green(rule.action) : c.red(rule.action); + const hitsStr = rule.hits > 0 ? c.bold(String(rule.hits)) : c.gray('0'); + lines.push(` ${paddedId}${actionStr} ${hitsStr} hits ${c.gray(rule.description)}`); + } + + // Denied requests + const denied = entries.filter(e => !e.isAllowed && e.url !== 'error:transaction-end-before-headers'); + if (denied.length > 0) { + lines.push(''); + lines.push(c.bold(`Denied Requests (${denied.length}):`)); + for (const entry of denied.slice(0, 20)) { + const ts = new Date(entry.timestamp * 1000).toISOString().slice(11, 23); + lines.push(` ${c.gray(ts)} ${c.red(entry.domain)} ${c.gray(`→ ${entry.matchedRuleId}`)}`); + } + if (denied.length > 20) { + lines.push(c.gray(` ...and ${denied.length - 20} more`)); + } + } + + lines.push(''); + return lines.join('\n'); +} + +/** + * Main handler for the `awf logs audit` subcommand + */ +export async function auditCommand(options: AuditCommandOptions): Promise { + const source = await discoverAndSelectSource(options.source, { + format: options.format, + shouldLog: (format) => format !== 'json', + }); + + // Load raw log entries + const entries = await loadAllLogs(source); + + if (entries.length === 0) { + logger.error('No log entries found.'); + process.exit(1); + } + + // Find policy manifest (uses shared discovery logic) + const manifest = findPolicyManifestForSource(source); + + if (!manifest) { + logger.error( + 'No policy-manifest.json found. The audit command requires a policy manifest.\n' + + 'Ensure you are using a version of awf that generates audit artifacts (--audit-dir).' + ); + process.exit(1); + } + + // Enrich entries with rule matching + let enriched = enrichWithPolicyRules(entries, manifest); + + // Apply filters + if (options.rule) { + enriched = enriched.filter(e => e.matchedRuleId === options.rule); + } + if (options.domain) { + const domainFilter = options.domain.toLowerCase(); + enriched = enriched.filter(e => e.domain.toLowerCase().includes(domainFilter)); + } + if (options.decision) { + const wantAllowed = options.decision === 'allowed'; + enriched = enriched.filter(e => e.isAllowed === wantAllowed); + } + + // Filter out benign operational entries + const meaningful = enriched.filter(e => e.url !== 'error:transaction-end-before-headers'); + + // Format and output + const colorize = !!(process.stdout.isTTY && options.format === 'pretty'); + let output: string; + + switch (options.format) { + case 'json': + output = formatAuditJson(meaningful); + break; + case 'markdown': + output = formatAuditMarkdown(meaningful, manifest); + break; + case 'pretty': + default: + output = formatAuditPretty(meaningful, manifest, colorize); + break; + } + + console.log(output); +} diff --git a/src/commands/logs-command-helpers.ts b/src/commands/logs-command-helpers.ts index 7a9b74c35..f96c120bf 100644 --- a/src/commands/logs-command-helpers.ts +++ b/src/commands/logs-command-helpers.ts @@ -1,16 +1,19 @@ /** - * Shared helper functions for log commands (stats and summary) + * Shared helper functions for log commands (stats, summary, audit) */ +import * as fs from 'fs'; +import * as path from 'path'; import { logger } from '../logger'; -import type { LogSource } from '../types'; +import type { LogSource, PolicyManifest } from '../types'; import { discoverLogSources, selectMostRecent, validateSource, } from '../logs/log-discovery'; -import { loadAndAggregate } from '../logs/log-aggregator'; +import { loadAndAggregate, loadAllLogs } from '../logs/log-aggregator'; import type { AggregatedStats } from '../logs/log-aggregator'; +import { enrichWithPolicyRules, computeRuleStats } from '../logs/audit-enricher'; /** * Options for determining which logs to show (based on log level) @@ -79,8 +82,41 @@ export async function discoverAndSelectSource( return source; } +/** + * Attempts to find a policy-manifest.json near a log source path. + * Returns null if not found. + */ +export function findPolicyManifestForSource(source: LogSource): PolicyManifest | null { + if (source.type === 'running' || !source.path) return null; + + const candidates = [ + path.join(source.path, 'policy-manifest.json'), + path.join(source.path, '..', 'audit', 'policy-manifest.json'), + source.path.replace(/squid-logs-/, 'awf-audit-').replace(/\/?$/, '/policy-manifest.json'), + ]; + + const auditDirEnv = process.env.AWF_AUDIT_DIR; + if (auditDirEnv) { + candidates.unshift(path.join(auditDirEnv, 'policy-manifest.json')); + } + + for (const candidate of candidates) { + try { + if (fs.existsSync(candidate)) { + const content = fs.readFileSync(candidate, 'utf-8'); + return JSON.parse(content) as PolicyManifest; + } + } catch { + // Skip + } + } + + return null; +} + /** * Loads and aggregates logs from a source, handling errors gracefully. + * Automatically enriches with policy rule stats when a manifest is available. * * @param source - Log source to load from * @returns Aggregated statistics @@ -89,7 +125,18 @@ export async function loadLogsWithErrorHandling( source: LogSource ): Promise { try { - return await loadAndAggregate(source); + const stats = await loadAndAggregate(source); + + // Try to enrich with policy rule stats + const manifest = findPolicyManifestForSource(source); + if (manifest) { + const entries = await loadAllLogs(source); + const enriched = enrichWithPolicyRules(entries, manifest); + stats.byRule = computeRuleStats(enriched, manifest); + logger.debug('Enriched stats with policy rule matching'); + } + + return stats; } catch (error) { logger.error(`Failed to load logs: ${error instanceof Error ? error.message : error}`); process.exit(1); diff --git a/src/docker-manager.ts b/src/docker-manager.ts index af9569012..c8406e4f1 100644 --- a/src/docker-manager.ts +++ b/src/docker-manager.ts @@ -5,7 +5,7 @@ import * as yaml from 'js-yaml'; import execa from 'execa'; import { DockerComposeConfig, WrapperConfig, BlockedTarget, API_PROXY_PORTS, API_PROXY_HEALTH_PORT } from './types'; import { logger } from './logger'; -import { generateSquidConfig } from './squid-config'; +import { generateSquidConfig, generatePolicyManifest } from './squid-config'; import { generateSessionCa, initSslDb, CaFiles, parseUrlPatterns, cleanupSslKeyMaterial, unmountSslTmpfs } from './ssl-bump'; const SQUID_PORT = 3128; @@ -1433,6 +1433,27 @@ export function generateDockerCompose( }; } +/** + * Redacts sensitive environment variables from a Docker Compose config for audit logging. + * Replaces values of env vars that look like secrets (tokens, keys, passwords) with "[REDACTED]". + */ +function redactDockerComposeSecrets(compose: DockerComposeConfig): DockerComposeConfig { + const sensitivePatterns = /(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API_KEY|_B64)$/i; + const redacted = JSON.parse(JSON.stringify(compose)) as DockerComposeConfig; + + for (const service of Object.values(redacted.services)) { + if (service.environment && typeof service.environment === 'object') { + for (const key of Object.keys(service.environment)) { + if (sensitivePatterns.test(key)) { + (service.environment as Record)[key] = '[REDACTED]'; + } + } + } + } + + return redacted; +} + /** * Writes configuration files to disk * Uses fixed network configuration (172.30.0.0/24) defined in host-iptables.ts @@ -1626,6 +1647,42 @@ export async function writeConfigs(config: WrapperConfig): Promise { // (like AWF_SQUID_CONFIG_B64) from being split across multiple lines fs.writeFileSync(dockerComposePath, yaml.dump(dockerCompose, { lineWidth: -1 }), { mode: 0o600 }); logger.debug(`Docker Compose config written to: ${dockerComposePath}`); + + // Write audit artifacts (config snapshots for post-run forensics) + const auditDir = config.auditDir || path.join(config.workDir, 'audit'); + if (!fs.existsSync(auditDir)) { + fs.mkdirSync(auditDir, { recursive: true, mode: 0o755 }); + } + + // Save squid.conf for audit (no secrets — just domain ACLs and proxy config) + fs.writeFileSync(path.join(auditDir, 'squid.conf'), squidConfig, { mode: 0o644 }); + + // Save redacted docker-compose.yml (strip env vars that may contain secrets) + const redactedCompose = redactDockerComposeSecrets(dockerCompose); + fs.writeFileSync( + path.join(auditDir, 'docker-compose.redacted.yml'), + yaml.dump(redactedCompose, { lineWidth: -1 }), + { mode: 0o644 } + ); + + // Generate and save policy manifest (structured description of all firewall rules) + const policyManifest = generatePolicyManifest({ + domains: config.allowedDomains, + blockedDomains: config.blockedDomains, + port: SQUID_PORT, + sslBump: config.sslBump, + enableHostAccess: config.enableHostAccess, + allowHostPorts: config.allowHostPorts, + enableDlp: config.enableDlp, + dnsServers: config.dnsServers, + }); + fs.writeFileSync( + path.join(auditDir, 'policy-manifest.json'), + JSON.stringify(policyManifest, null, 2), + { mode: 0o644 } + ); + + logger.debug(`Audit artifacts written to: ${auditDir}`); } /** @@ -1930,7 +1987,25 @@ export async function stopContainers(workDir: string, keepContainers: boolean): * @param keepFiles - If true, skip cleanup and keep files * @param proxyLogsDir - Optional custom directory where Squid proxy logs were written directly */ -export async function cleanup(workDir: string, keepFiles: boolean, proxyLogsDir?: string): Promise { +/** + * Copies the iptables audit dump from the init-signal volume to the audit directory. + * Must be called BEFORE stopContainers() because `docker compose down -v` destroys + * the init-signal volume. + */ +export function preserveIptablesAudit(workDir: string, auditDir?: string): void { + const iptablesAuditSrc = path.join(workDir, 'init-signal', 'iptables-audit.txt'); + const targetAuditDir = auditDir || path.join(workDir, 'audit'); + if (fs.existsSync(iptablesAuditSrc) && fs.existsSync(targetAuditDir)) { + try { + fs.copyFileSync(iptablesAuditSrc, path.join(targetAuditDir, 'iptables-audit.txt')); + logger.debug('Copied iptables audit state to audit directory'); + } catch (error) { + logger.debug('Could not copy iptables audit file:', error); + } + } +} + +export async function cleanup(workDir: string, keepFiles: boolean, proxyLogsDir?: string, auditDir?: string): Promise { if (keepFiles) { logger.debug(`Keeping temporary files in: ${workDir}`); return; @@ -2015,6 +2090,32 @@ export async function cleanup(workDir: string, keepFiles: boolean, proxyLogsDir? } } + // Preserve audit artifacts + if (auditDir) { + // User-specified audit dir: just fix permissions + if (fs.existsSync(auditDir)) { + try { + execa.sync('chmod', ['-R', 'a+rX', auditDir]); + logger.info(`Audit artifacts available at: ${auditDir}`); + } catch (error) { + logger.debug('Could not fix audit dir permissions:', error); + } + } + } else { + // Default: move from workDir/audit to timestamped /tmp directory + const defaultAuditDir = path.join(workDir, 'audit'); + const auditDestination = path.join(os.tmpdir(), `awf-audit-${timestamp}`); + if (fs.existsSync(defaultAuditDir) && fs.readdirSync(defaultAuditDir).length > 0) { + try { + fs.renameSync(defaultAuditDir, auditDestination); + execa.sync('chmod', ['-R', 'a+rX', auditDestination]); + logger.info(`Audit artifacts preserved at: ${auditDestination}`); + } catch (error) { + logger.debug('Could not preserve audit artifacts:', error); + } + } + } + // Securely wipe SSL key material before deleting workDir cleanupSslKeyMaterial(workDir); diff --git a/src/logs/audit-enricher.test.ts b/src/logs/audit-enricher.test.ts new file mode 100644 index 000000000..ad114077f --- /dev/null +++ b/src/logs/audit-enricher.test.ts @@ -0,0 +1,340 @@ +import { enrichWithPolicyRules, computeRuleStats, EnrichedLogEntry } from './audit-enricher'; +import { ParsedLogEntry, PolicyManifest, PolicyRule } from '../types'; + +function makeEntry(overrides: Partial = {}): ParsedLogEntry { + return { + timestamp: 1700000000.000, + clientIp: '172.30.0.20', + clientPort: '39748', + host: 'github.com:443', + destIp: '140.82.114.22', + destPort: '443', + protocol: '1.1', + method: 'CONNECT', + statusCode: 200, + decision: 'TCP_TUNNEL:HIER_DIRECT', + url: 'github.com:443', + userAgent: 'curl/7.81.0', + domain: 'github.com', + isAllowed: true, + isHttps: true, + ...overrides, + }; +} + +function makeManifest(rules: PolicyRule[]): PolicyManifest { + return { + version: 1, + generatedAt: '2024-01-01T00:00:00.000Z', + rules, + dangerousPorts: [22, 3306], + dnsServers: ['8.8.8.8'], + sslBumpEnabled: false, + dlpEnabled: false, + hostAccessEnabled: false, + allowHostPorts: null, + }; +} + +describe('enrichWithPolicyRules', () => { + it('should match allowed request to allow-both-plain rule', () => { + const manifest = makeManifest([ + { + id: 'allow-both-plain', + order: 1, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow HTTP and HTTPS traffic to these domains', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ domain: 'github.com', isAllowed: true })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched).toHaveLength(1); + expect(enriched[0].matchedRuleId).toBe('allow-both-plain'); + }); + + it('should match subdomain to parent domain rule', () => { + const manifest = makeManifest([ + { + id: 'allow-both-plain', + order: 1, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ domain: 'api.github.com', isAllowed: true })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched[0].matchedRuleId).toBe('allow-both-plain'); + }); + + it('should match denied request to default deny rule', () => { + const manifest = makeManifest([ + { + id: 'allow-both-plain', + order: 1, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ + domain: 'evil.com', + isAllowed: false, + statusCode: 403, + decision: 'TCP_DENIED:HIER_NONE', + })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched[0].matchedRuleId).toBe('deny-default'); + }); + + it('should match blocked domain to deny-blocked rule before allow', () => { + const manifest = makeManifest([ + { + id: 'deny-blocked-plain', + order: 1, + action: 'deny', + aclName: 'blocked_domains', + protocol: 'both', + domains: ['.evil.com'], + description: 'Deny blocked', + }, + { + id: 'allow-both-plain', + order: 2, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.example.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 3, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ + domain: 'evil.com', + isAllowed: false, + statusCode: 403, + decision: 'TCP_DENIED:HIER_NONE', + })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched[0].matchedRuleId).toBe('deny-blocked-plain'); + }); + + it('should match regex patterns', () => { + const manifest = makeManifest([ + { + id: 'allow-both-regex', + order: 1, + action: 'allow', + aclName: 'allowed_domains_regex', + protocol: 'both', + domains: ['^.*\\.github\\.com$'], + description: 'Allow wildcard', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ domain: 'api.github.com', isAllowed: true })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched[0].matchedRuleId).toBe('allow-both-regex'); + }); + + it('should match raw IP deny rules (dst_ipv4 with regex patterns)', () => { + const manifest = makeManifest([ + { + id: 'deny-raw-ipv4', + order: 1, + action: 'deny', + aclName: 'dst_ipv4', + protocol: 'both', + domains: ['^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$'], + description: 'Deny raw IPv4', + }, + { + id: 'allow-both-plain', + order: 2, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 3, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries = [makeEntry({ + domain: '93.184.216.34', + isAllowed: false, + statusCode: 403, + decision: 'TCP_DENIED:HIER_NONE', + })]; + const enriched = enrichWithPolicyRules(entries, manifest); + + expect(enriched[0].matchedRuleId).toBe('deny-raw-ipv4'); + }); + + it('should respect protocol-specific rules', () => { + const manifest = makeManifest([ + { + id: 'allow-https-only-plain', + order: 1, + action: 'allow', + aclName: 'allowed_https_only', + protocol: 'https', + domains: ['.secure.com'], + description: 'HTTPS only', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + // HTTPS request should match the HTTPS-only rule + const httpsEntry = makeEntry({ domain: 'secure.com', isHttps: true, isAllowed: true }); + const enrichedHttps = enrichWithPolicyRules([httpsEntry], manifest); + expect(enrichedHttps[0].matchedRuleId).toBe('allow-https-only-plain'); + + // HTTP request should NOT match the HTTPS-only rule, falls to deny-default + const httpEntry = makeEntry({ domain: 'secure.com', isHttps: false, method: 'GET', isAllowed: false }); + const enrichedHttp = enrichWithPolicyRules([httpEntry], manifest); + expect(enrichedHttp[0].matchedRuleId).toBe('deny-default'); + }); +}); + +describe('computeRuleStats', () => { + it('should count hits per rule', () => { + const manifest = makeManifest([ + { + id: 'allow-both-plain', + order: 1, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const entries: EnrichedLogEntry[] = [ + { ...makeEntry({ domain: 'github.com' }), matchedRuleId: 'allow-both-plain', matchReason: '' }, + { ...makeEntry({ domain: 'api.github.com' }), matchedRuleId: 'allow-both-plain', matchReason: '' }, + { ...makeEntry({ domain: 'evil.com', isAllowed: false }), matchedRuleId: 'deny-default', matchReason: '' }, + ]; + + const stats = computeRuleStats(entries, manifest); + + expect(stats).toHaveLength(2); + expect(stats.find(r => r.ruleId === 'allow-both-plain')?.hits).toBe(2); + expect(stats.find(r => r.ruleId === 'deny-default')?.hits).toBe(1); + }); + + it('should report 0 hits for unused rules', () => { + const manifest = makeManifest([ + { + id: 'allow-both-plain', + order: 1, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: ['.github.com'], + description: 'Allow', + }, + { + id: 'deny-default', + order: 2, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Default deny', + }, + ]); + + const stats = computeRuleStats([], manifest); + + expect(stats).toHaveLength(2); + expect(stats[0].hits).toBe(0); + expect(stats[1].hits).toBe(0); + }); +}); diff --git a/src/logs/audit-enricher.ts b/src/logs/audit-enricher.ts new file mode 100644 index 000000000..31bc59ac3 --- /dev/null +++ b/src/logs/audit-enricher.ts @@ -0,0 +1,166 @@ +/** + * Enriches parsed log entries with policy rule matching information. + * + * Given a PolicyManifest and parsed log entries, this module determines which + * firewall rule caused each allow/deny decision by replaying the ACL evaluation + * order. Squid evaluates http_access rules top-to-bottom, applying the first match. + */ + +import { ParsedLogEntry, PolicyManifest, PolicyRule } from '../types'; + +/** + * A log entry enriched with the policy rule that matched it. + */ +export interface EnrichedLogEntry extends ParsedLogEntry { + /** ID of the policy rule that matched (e.g., "allow-both-plain", "deny-default") */ + matchedRuleId: string; + /** Human-readable reason for the decision */ + matchReason: string; +} + +/** + * Per-rule hit statistics. + */ +export interface RuleStats { + ruleId: string; + description: string; + action: 'allow' | 'deny'; + hits: number; +} + +/** + * Checks whether a domain matches any entry in a rule's domain list. + * + * For plain domain rules (dstdomain), domains are listed as ".github.com" + * which matches both "github.com" and "*.github.com". + * + * For regex rules (dstdom_regex), domains are listed as regex patterns. + */ +function domainMatchesRule(domain: string, rule: PolicyRule): boolean { + if (!domain || domain === '-') return false; + + const lowerDomain = domain.toLowerCase(); + + // Detect regex rules: either the ACL name contains "regex", or the domain + // entries contain regex metacharacters (e.g., dst_ipv4 uses ^[0-9]+ patterns) + const isRegexRule = rule.aclName.includes('regex') || + rule.domains.some(d => /[\\^$*+?{}()[\]|]/.test(d)); + + for (const entry of rule.domains) { + if (isRegexRule) { + // Regex match + try { + const re = new RegExp(entry, 'i'); + if (re.test(lowerDomain)) return true; + } catch { + // Invalid regex, skip + } + } else { + // Plain domain match: ".github.com" matches "github.com" and "api.github.com" + const aclDomain = entry.toLowerCase(); + if (aclDomain.startsWith('.')) { + const baseDomain = aclDomain.slice(1); + if (lowerDomain === baseDomain || lowerDomain.endsWith(aclDomain)) { + return true; + } + } else { + if (lowerDomain === aclDomain) return true; + } + } + } + + return false; +} + +/** + * Checks whether a rule's protocol constraint matches the request. + */ +function protocolMatches(rule: PolicyRule, isHttps: boolean): boolean { + if (rule.protocol === 'both') return true; + if (rule.protocol === 'https' && isHttps) return true; + if (rule.protocol === 'http' && !isHttps) return true; + return false; +} + +/** + * Finds the first matching policy rule for a log entry by replaying + * the http_access evaluation order. + */ +function findMatchingRule(entry: ParsedLogEntry, rules: PolicyRule[]): PolicyRule | null { + for (const rule of rules) { + if (!protocolMatches(rule, entry.isHttps)) continue; + + // The default deny rule (aclName: "all") matches everything + if (rule.aclName === 'all') return rule; + + // For deny rules with specific domains, match if domain is in the list + if (rule.action === 'deny' && domainMatchesRule(entry.domain, rule)) { + return rule; + } + + // For allow rules, the Squid logic is: + // "http_access deny !allowed_domains" means: deny if NOT in allowed_domains + // So an allow rule matches if the domain IS in the list + if (rule.action === 'allow' && domainMatchesRule(entry.domain, rule)) { + return rule; + } + } + + return null; +} + +/** + * Enriches parsed log entries with policy rule matching. + * + * For each entry, replays the ACL evaluation order from the manifest + * to determine which rule caused the allow/deny decision. + */ +export function enrichWithPolicyRules( + entries: ParsedLogEntry[], + manifest: PolicyManifest +): EnrichedLogEntry[] { + // Sort rules by evaluation order + const sortedRules = [...manifest.rules].sort((a, b) => a.order - b.order); + + return entries.map(entry => { + const matchedRule = findMatchingRule(entry, sortedRules); + + if (matchedRule) { + return { + ...entry, + matchedRuleId: matchedRule.id, + matchReason: matchedRule.description, + }; + } + + // Fallback: no rule matched (shouldn't happen with a deny-default rule) + return { + ...entry, + matchedRuleId: 'unknown', + matchReason: entry.isAllowed ? 'Allowed (rule not identified)' : 'Denied (rule not identified)', + }; + }); +} + +/** + * Computes per-rule hit statistics from enriched log entries. + */ +export function computeRuleStats( + enrichedEntries: EnrichedLogEntry[], + manifest: PolicyManifest +): RuleStats[] { + const hitCounts = new Map(); + + for (const entry of enrichedEntries) { + // Skip benign operational entries + if (entry.url === 'error:transaction-end-before-headers') continue; + hitCounts.set(entry.matchedRuleId, (hitCounts.get(entry.matchedRuleId) || 0) + 1); + } + + return manifest.rules.map(rule => ({ + ruleId: rule.id, + description: rule.description, + action: rule.action, + hits: hitCounts.get(rule.id) || 0, + })); +} diff --git a/src/logs/log-aggregator.ts b/src/logs/log-aggregator.ts index 69b721d97..7cc9bee63 100644 --- a/src/logs/log-aggregator.ts +++ b/src/logs/log-aggregator.ts @@ -6,7 +6,7 @@ import * as fs from 'fs'; import * as path from 'path'; import execa from 'execa'; import { LogSource, ParsedLogEntry } from '../types'; -import { parseLogLine } from './log-parser'; +import { parseLogLine, parseAuditJsonlLine } from './log-parser'; import { logger } from '../logger'; /** @@ -39,6 +39,33 @@ export interface AggregatedStats { byDomain: Map; /** Time range of the logs (null if no entries) */ timeRange: { start: number; end: number } | null; + /** Per-rule hit statistics (populated when policy manifest is available) */ + byRule?: import('./audit-enricher').RuleStats[]; +} + +/** + * Parses lines of text into log entries using the given parser function. + */ +function parseLines( + content: string, + parser: (line: string) => ParsedLogEntry | null +): ParsedLogEntry[] { + const entries: ParsedLogEntry[] = []; + const lines = content.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + + const entry = parser(trimmed); + if (entry) { + entries.push(entry); + } else { + logger.debug(`Failed to parse log line: ${trimmed}`); + } + } + + return entries; } /** @@ -143,38 +170,36 @@ export async function loadAllLogs(source: LogSource): Promise return []; } } else { - // Read from file + // Read from file — prefer audit.jsonl (structured) over access.log (text) if (!source.path) { throw new Error('Path is required for preserved log source'); } - const filePath = path.join(source.path, 'access.log'); - logger.debug(`Loading logs from file: ${filePath}`); - if (!fs.existsSync(filePath)) { - logger.debug(`Log file not found: ${filePath}`); - return []; + const jsonlPath = path.join(source.path, 'audit.jsonl'); + const textPath = path.join(source.path, 'access.log'); + + // Try JSONL first, fall back to text format + if (fs.existsSync(jsonlPath)) { + const jsonlContent = fs.readFileSync(jsonlPath, 'utf-8'); + const jsonlEntries = parseLines(jsonlContent, parseAuditJsonlLine); + if (jsonlEntries.length > 0) { + logger.debug(`Loaded ${jsonlEntries.length} entries from JSONL: ${jsonlPath}`); + return jsonlEntries; + } + logger.debug(`JSONL file had no parseable entries, falling back to text format`); } - content = fs.readFileSync(filePath, 'utf-8'); - } - - // Parse all lines - const entries: ParsedLogEntry[] = []; - const lines = content.split('\n'); - - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) continue; - - const entry = parseLogLine(trimmed); - if (entry) { - entries.push(entry); + if (fs.existsSync(textPath)) { + content = fs.readFileSync(textPath, 'utf-8'); + logger.debug(`Loading logs from text: ${textPath}`); } else { - logger.debug(`Failed to parse log line: ${trimmed}`); + logger.debug(`No log files found in: ${source.path}`); + return []; } } - return entries; + // Parse all lines (for running container source or text file fallback) + return parseLines(content, parseLogLine); } /** diff --git a/src/logs/log-parser.test.ts b/src/logs/log-parser.test.ts index 50f4e7379..75ddc2993 100644 --- a/src/logs/log-parser.test.ts +++ b/src/logs/log-parser.test.ts @@ -2,7 +2,7 @@ * Unit tests for log-parser.ts */ -import { parseLogLine, extractDomain, extractPort } from './log-parser'; +import { parseLogLine, extractDomain, extractPort, parseAuditJsonlLine } from './log-parser'; describe('log-parser', () => { describe('parseLogLine', () => { @@ -244,4 +244,51 @@ describe('log-parser', () => { expect(extractPort('api.github.com:abc', 'CONNECT')).toBeUndefined(); }); }); + + describe('parseAuditJsonlLine', () => { + it('should parse a valid JSONL CONNECT entry', () => { + const line = '{"ts":1761074374.646,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.114.22:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"}'; + const entry = parseAuditJsonlLine(line); + + expect(entry).not.toBeNull(); + expect(entry!.timestamp).toBeCloseTo(1761074374.646); + expect(entry!.clientIp).toBe('172.30.0.20'); + expect(entry!.method).toBe('CONNECT'); + expect(entry!.statusCode).toBe(200); + expect(entry!.decision).toBe('TCP_TUNNEL'); + expect(entry!.domain).toBe('api.github.com'); + expect(entry!.isAllowed).toBe(true); + expect(entry!.isHttps).toBe(true); + }); + + it('should parse a denied JSONL entry', () => { + const line = '{"ts":1760994429.358,"client":"172.30.0.20","host":"evil.com:443","dest":"-:-","method":"CONNECT","status":403,"decision":"TCP_DENIED","url":"evil.com:443"}'; + const entry = parseAuditJsonlLine(line); + + expect(entry).not.toBeNull(); + expect(entry!.isAllowed).toBe(false); + expect(entry!.statusCode).toBe(403); + expect(entry!.domain).toBe('evil.com'); + }); + + it('should parse a HTTP GET entry', () => { + const line = '{"ts":1700000000.000,"client":"172.30.0.20","host":"example.com","dest":"93.184.216.34:80","method":"GET","status":200,"decision":"TCP_MISS","url":"http://example.com/"}'; + const entry = parseAuditJsonlLine(line); + + expect(entry).not.toBeNull(); + expect(entry!.isHttps).toBe(false); + expect(entry!.isAllowed).toBe(true); + expect(entry!.domain).toBe('example.com'); + }); + + it('should return null for empty lines', () => { + expect(parseAuditJsonlLine('')).toBeNull(); + expect(parseAuditJsonlLine(' ')).toBeNull(); + }); + + it('should return null for invalid JSON', () => { + expect(parseAuditJsonlLine('not json')).toBeNull(); + expect(parseAuditJsonlLine('{broken')).toBeNull(); + }); + }); }); diff --git a/src/logs/log-parser.ts b/src/logs/log-parser.ts index f7e0d58ee..a3affe0fe 100644 --- a/src/logs/log-parser.ts +++ b/src/logs/log-parser.ts @@ -157,3 +157,57 @@ export function extractPort(url: string, method: string): string | undefined { } return undefined; } + +/** + * Parses a single line from the JSONL audit log (audit.jsonl). + * + * Format: {"ts":1761074374.646,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.114.22:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} + * + * @param line - Raw JSONL line + * @returns Parsed log entry or null if parsing failed + */ +export function parseAuditJsonlLine(line: string): ParsedLogEntry | null { + const trimmed = line.trim(); + if (!trimmed) return null; + + try { + const obj = JSON.parse(trimmed); + + const method = obj.method || ''; + const isHttps = method === 'CONNECT'; + const decision = obj.decision || ''; + const isAllowed = decision.startsWith('TCP_TUNNEL') || decision.startsWith('TCP_MISS'); + + // Parse dest into IP and port + const destStr = obj.dest || '-:-'; + const destParts = destStr.split(':'); + const destIp = destParts[0] || '-'; + const destPort = destParts[1] || '-'; + + // Extract domain + const url = obj.url || ''; + const host = obj.host || '-'; + const domain = extractDomain(url, host, method); + + return { + timestamp: obj.ts || 0, + clientIp: obj.client || '-', + clientPort: '-', // Not in JSONL format + host, + destIp, + destPort, + protocol: '-', // Not in JSONL format + method, + statusCode: obj.status || 0, + decision, + url, + userAgent: '-', // Not in JSONL format (intentionally omitted) + domain, + isAllowed, + isHttps, + }; + } catch { + // JSON parse failed — likely a line with unescaped characters + return null; + } +} diff --git a/src/logs/stats-formatter.ts b/src/logs/stats-formatter.ts index 5a60f8620..ad76e459f 100644 --- a/src/logs/stats-formatter.ts +++ b/src/logs/stats-formatter.ts @@ -22,7 +22,7 @@ export function formatStatsJson(stats: AggregatedStats): string { }; } - const output = { + const output: Record = { totalRequests: stats.totalRequests, allowedRequests: stats.allowedRequests, deniedRequests: stats.deniedRequests, @@ -31,6 +31,10 @@ export function formatStatsJson(stats: AggregatedStats): string { byDomain, }; + if (stats.byRule) { + output.byRule = stats.byRule; + } + return JSON.stringify(output, null, 2); } @@ -87,6 +91,22 @@ export function formatStatsMarkdown(stats: AggregatedStats): string { lines.push('No firewall activity detected.'); } + // Policy rules section (when available) + if (stats.byRule && stats.byRule.length > 0) { + lines.push(''); + lines.push('
'); + lines.push('Policy Rules\n'); + lines.push('| Rule | Action | Hits | Description |'); + lines.push('|------|--------|------|-------------|'); + + for (const rule of stats.byRule) { + const actionEmoji = rule.action === 'allow' ? '✅' : '🚫'; + lines.push(`| ${rule.ruleId} | ${actionEmoji} ${rule.action} | ${rule.hits} | ${rule.description} |`); + } + + lines.push('\n
'); + } + lines.push('\n\n'); return lines.join('\n'); @@ -167,6 +187,19 @@ export function formatStatsPretty( } } + // Policy rules section + if (stats.byRule && stats.byRule.length > 0) { + lines.push(c.bold('Policy Rules:')); + const maxIdLen = Math.max(...stats.byRule.map(r => r.ruleId.length)); + for (const rule of stats.byRule) { + const paddedId = rule.ruleId.padEnd(maxIdLen + 2); + const actionStr = rule.action === 'allow' ? c.green(rule.action) : c.red(rule.action); + const hitsStr = rule.hits > 0 ? String(rule.hits) : c.gray('0'); + lines.push(` ${paddedId}${actionStr} ${hitsStr} hits ${c.gray(rule.description)}`); + } + lines.push(''); + } + lines.push(''); return lines.join('\n'); } diff --git a/src/squid-config.test.ts b/src/squid-config.test.ts index f4427ab29..333d73f8e 100644 --- a/src/squid-config.test.ts +++ b/src/squid-config.test.ts @@ -1,4 +1,4 @@ -import { generateSquidConfig } from './squid-config'; +import { generateSquidConfig, generatePolicyManifest } from './squid-config'; import { SquidConfig } from './types'; // Pattern constant for the safer domain character class (matches the implementation) @@ -1661,3 +1661,145 @@ describe('DLP Integration', () => { expect(result).toContain('ssl_bump'); }); }); + +describe('generatePolicyManifest', () => { + const defaultPort = 3128; + + it('should generate manifest with basic allowed domains', () => { + const manifest = generatePolicyManifest({ + domains: ['github.com', 'api.github.com'], + port: defaultPort, + }); + + expect(manifest.version).toBe(1); + expect(manifest.generatedAt).toBeDefined(); + expect(manifest.sslBumpEnabled).toBe(false); + expect(manifest.dlpEnabled).toBe(false); + + // Should have allow-both-plain and deny-default rules + const allowRule = manifest.rules.find(r => r.id === 'allow-both-plain'); + expect(allowRule).toBeDefined(); + expect(allowRule!.action).toBe('allow'); + expect(allowRule!.protocol).toBe('both'); + expect(allowRule!.domains).toContain('.github.com'); + + const denyRule = manifest.rules.find(r => r.id === 'deny-default'); + expect(denyRule).toBeDefined(); + expect(denyRule!.action).toBe('deny'); + }); + + it('should include blocked domains as deny rules with precedence', () => { + const manifest = generatePolicyManifest({ + domains: ['github.com'], + blockedDomains: ['evil.com'], + port: defaultPort, + }); + + const blockedRule = manifest.rules.find(r => r.id === 'deny-blocked-plain'); + expect(blockedRule).toBeDefined(); + // Blocked domains come after port safety and raw IP rules but before allow rules + expect(blockedRule!.action).toBe('deny'); + expect(blockedRule!.domains).toContain('.evil.com'); + + const allowRule = manifest.rules.find(r => r.id === 'allow-both-plain'); + expect(allowRule).toBeDefined(); + expect(allowRule!.order).toBeGreaterThan(blockedRule!.order); + }); + + it('should handle protocol-specific domains', () => { + const manifest = generatePolicyManifest({ + domains: ['http://httponly.com', 'https://httpsonly.com', 'both.com'], + port: defaultPort, + }); + + const httpRule = manifest.rules.find(r => r.id === 'allow-http-only-plain'); + expect(httpRule).toBeDefined(); + expect(httpRule!.protocol).toBe('http'); + + const httpsRule = manifest.rules.find(r => r.id === 'allow-https-only-plain'); + expect(httpsRule).toBeDefined(); + expect(httpsRule!.protocol).toBe('https'); + + const bothRule = manifest.rules.find(r => r.id === 'allow-both-plain'); + expect(bothRule).toBeDefined(); + expect(bothRule!.protocol).toBe('both'); + }); + + it('should handle wildcard domains as regex rules', () => { + const manifest = generatePolicyManifest({ + domains: ['*.github.com'], + port: defaultPort, + }); + + const regexRule = manifest.rules.find(r => r.id === 'allow-both-regex'); + expect(regexRule).toBeDefined(); + expect(regexRule!.aclName).toBe('allowed_domains_regex'); + expect(regexRule!.domains.length).toBeGreaterThan(0); + }); + + it('should always end with deny-default rule', () => { + const manifest = generatePolicyManifest({ + domains: ['github.com'], + port: defaultPort, + }); + + const lastRule = manifest.rules[manifest.rules.length - 1]; + expect(lastRule.id).toBe('deny-default'); + expect(lastRule.action).toBe('deny'); + expect(lastRule.aclName).toBe('all'); + }); + + it('should include dangerous ports list', () => { + const manifest = generatePolicyManifest({ + domains: ['github.com'], + port: defaultPort, + }); + + expect(manifest.dangerousPorts).toContain(22); + expect(manifest.dangerousPorts).toContain(3306); + expect(manifest.dangerousPorts).toContain(5432); + }); + + it('should reflect config flags', () => { + const manifest = generatePolicyManifest({ + domains: ['github.com'], + port: defaultPort, + sslBump: true, + enableDlp: true, + enableHostAccess: true, + allowHostPorts: '3000,8080', + dnsServers: ['1.1.1.1'], + }); + + expect(manifest.sslBumpEnabled).toBe(true); + expect(manifest.dlpEnabled).toBe(true); + expect(manifest.hostAccessEnabled).toBe(true); + expect(manifest.allowHostPorts).toBe('3000,8080'); + expect(manifest.dnsServers).toEqual(['1.1.1.1']); + }); + + it('should maintain consistent rule ordering with generateSquidConfig', () => { + // The manifest rule order should mirror the http_access rule order + const config: SquidConfig = { + domains: ['github.com', 'http://httponly.com'], + blockedDomains: ['evil.com'], + port: defaultPort, + }; + + const manifest = generatePolicyManifest(config); + const squidConfig = generateSquidConfig(config); + + // Port safety and raw IP rules come first, then blocked domains, then allow rules + const portRule = manifest.rules.find(r => r.id === 'deny-unsafe-ports'); + const blockedRule = manifest.rules.find(r => r.id === 'deny-blocked-plain'); + expect(portRule!.order).toBeLessThan(blockedRule!.order); + expect(squidConfig.indexOf('deny blocked_domains')).toBeLessThan( + squidConfig.indexOf('allow !CONNECT') + ); + + // HTTP-only should come before the catch-all deny + const httpRule = manifest.rules.find(r => r.id === 'allow-http-only-plain'); + const denyRule = manifest.rules.find(r => r.id === 'deny-default'); + expect(httpRule!.order).toBeLessThan(denyRule!.order); + }); +}); diff --git a/src/squid-config.ts b/src/squid-config.ts index 3be197830..ba540d4a3 100644 --- a/src/squid-config.ts +++ b/src/squid-config.ts @@ -1,4 +1,4 @@ -import { SquidConfig } from './types'; +import { SquidConfig, PolicyManifest, PolicyRule } from './types'; import { parseDomainList, isDomainMatchedByPattern, @@ -82,6 +82,40 @@ function groupPatternsByProtocol(patterns: DomainPattern[]): PatternsByProtocol return result; } +/** + * Shared domain parsing: validates, deduplicates, filters, and groups domains by protocol. + * Used by both generateSquidConfig and generatePolicyManifest to ensure consistent logic. + */ +function parseDomainConfig(domains: string[]): { + domainsByProto: DomainsByProtocol; + patternsByProto: PatternsByProtocol; + patterns: DomainPattern[]; +} { + const { plainDomains, patterns } = parseDomainList(domains); + + // Remove redundant plain subdomains within same protocol + const uniquePlainDomains = plainDomains.filter((entry, index, arr) => { + return !arr.some((other, otherIndex) => { + if (index === otherIndex) return false; + if (entry.domain === other.domain || !entry.domain.endsWith('.' + other.domain)) { + return false; + } + return other.protocol === 'both' || other.protocol === entry.protocol; + }); + }); + + // Remove plain domains already covered by wildcard patterns + const filteredPlainDomains = uniquePlainDomains.filter(entry => { + return !isDomainMatchedByPattern(entry, patterns); + }); + + return { + domainsByProto: groupDomainsByProtocol(filteredPlainDomains), + patternsByProto: groupPatternsByProtocol(patterns), + patterns, + }; +} + /** * Generates SSL Bump configuration section for HTTPS content inspection * @@ -208,34 +242,8 @@ ${urlAclSection}${urlAccessRules}`; export function generateSquidConfig(config: SquidConfig): string { const { domains, blockedDomains, port, sslBump, caFiles, sslDbPath, urlPatterns, enableHostAccess, allowHostPorts, enableDlp, dnsServers } = config; - // Parse domains into plain domains and wildcard patterns - // Note: parseDomainList extracts and preserves protocol info from prefixes (http://, https://) - // This also validates all inputs and throws on invalid patterns - const { plainDomains, patterns } = parseDomainList(domains); - - // Remove redundant plain subdomains within same protocol - // (e.g., if github.com with 'both' is present, api.github.com with 'both' is redundant) - const uniquePlainDomains = plainDomains.filter((entry, index, arr) => { - // Check if this domain is a subdomain of another plain domain with compatible protocol - return !arr.some((other, otherIndex) => { - if (index === otherIndex) return false; - // Check if this domain is a subdomain of other - if (entry.domain === other.domain || !entry.domain.endsWith('.' + other.domain)) { - return false; - } - // Subdomain is only redundant if parent has same or broader protocol - return other.protocol === 'both' || other.protocol === entry.protocol; - }); - }); - - // Remove plain domains that are already covered by wildcard patterns - const filteredPlainDomains = uniquePlainDomains.filter(entry => { - return !isDomainMatchedByPattern(entry, patterns); - }); - - // Group domains and patterns by protocol - const domainsByProto = groupDomainsByProtocol(filteredPlainDomains); - const patternsByProto = groupPatternsByProtocol(patterns); + // Parse, deduplicate, and group domains by protocol (shared logic) + const { domainsByProto, patternsByProto } = parseDomainConfig(domains); // Generate ACL entries const aclLines: string[] = []; @@ -524,10 +532,16 @@ pid_filename /var/run/squid/squid.pid # Note: For CONNECT requests (HTTPS), the domain is in the URL field logformat firewall_detailed %ts.%03tu %>a:%>p %{Host}>h %Hs %Ss:%Sh %ru "%{User-Agent}>h" +# Structured JSONL audit log for machine-readable analysis +# Note: Squid logformat does not JSON-escape strings, so fields like User-Agent +# could break JSON parsing. We omit User-Agent to reduce breakage risk. +logformat audit_jsonl {"ts":%ts.%03tu,"client":"%>a","host":"%{Host}>h","dest":"%Hs,"decision":"%Ss","url":"%ru"} + # Access log and cache configuration # Don't log healthcheck probes from localhost (using ACL filter on access_log) acl healthcheck_localhost src 127.0.0.1 ::1 access_log /var/log/squid/access.log firewall_detailed !healthcheck_localhost +access_log /var/log/squid/audit.jsonl audit_jsonl !healthcheck_localhost cache_log /var/log/squid/cache.log cache deny all @@ -614,3 +628,197 @@ shutdown_lifetime 0 seconds # debug_options ALL,1 33,2 `; } + +/** + * Generates a structured policy manifest describing all firewall rules in effect. + * + * The manifest mirrors the http_access rule ordering in generateSquidConfig() so + * that post-hoc log analysis can deterministically replay which rule matched each + * request by evaluating rules in order and stopping at the first match. + */ +export function generatePolicyManifest(config: SquidConfig): PolicyManifest { + const { domains, blockedDomains, sslBump, enableHostAccess, allowHostPorts, enableDlp, dnsServers } = config; + + // Parse, deduplicate, and group domains by protocol (shared logic with generateSquidConfig) + const { domainsByProto, patternsByProto } = parseDomainConfig(domains); + + const rules: PolicyRule[] = []; + let order = 0; + + // --- Port safety rules (evaluated first in Squid) --- + rules.push({ + id: 'deny-unsafe-ports', + order: ++order, + action: 'deny', + aclName: '!Safe_ports', + protocol: 'both', + domains: [], + description: 'Deny requests to ports not in Safe_ports ACL (only 80, 443, and user-specified ports allowed)', + }); + rules.push({ + id: 'deny-connect-unsafe-ports', + order: ++order, + action: 'deny', + aclName: 'CONNECT !Safe_ports', + protocol: 'https', + domains: [], + description: 'Deny CONNECT (HTTPS) to ports not in Safe_ports ACL', + }); + + // --- Raw IP blocking --- + rules.push({ + id: 'deny-raw-ipv4', + order: ++order, + action: 'deny', + aclName: 'dst_ipv4', + protocol: 'both', + domains: ['^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$'], + description: 'Deny requests to raw IPv4 addresses (bypasses domain filtering)', + }); + rules.push({ + id: 'deny-raw-ipv6', + order: ++order, + action: 'deny', + aclName: 'dst_ipv6', + protocol: 'both', + domains: ['^\\[?[0-9a-fA-F:]+\\]?$'], + description: 'Deny requests to raw IPv6 addresses (bypasses domain filtering)', + }); + + // --- DLP rules (if enabled) --- + if (enableDlp) { + rules.push({ + id: 'deny-dlp', + order: ++order, + action: 'deny', + aclName: 'dlp_blocked', + protocol: 'both', + domains: [], + description: 'Deny requests containing credential patterns in URLs (DLP)', + }); + } + + // --- Blocked domains --- + if (blockedDomains && blockedDomains.length > 0) { + const normalizedBlocked = blockedDomains.map(d => d.replace(/^https?:\/\//, '').replace(/\/$/, '')); + const { plainDomains: blockedPlain, patterns: blockedPatterns } = parseDomainList(normalizedBlocked); + + if (blockedPlain.length > 0) { + rules.push({ + id: 'deny-blocked-plain', + order: ++order, + action: 'deny', + aclName: 'blocked_domains', + protocol: 'both', + domains: blockedPlain.map(e => formatDomainForSquid(e.domain)), + description: 'Deny requests to explicitly blocked domains', + }); + } + + if (blockedPatterns.length > 0) { + rules.push({ + id: 'deny-blocked-regex', + order: ++order, + action: 'deny', + aclName: 'blocked_domains_regex', + protocol: 'both', + domains: blockedPatterns.map(p => p.regex), + description: 'Deny requests to explicitly blocked domain patterns', + }); + } + } + + // --- Protocol-specific allow rules --- + if (domainsByProto.http.length > 0) { + rules.push({ + id: 'allow-http-only-plain', + order: ++order, + action: 'allow', + aclName: 'allowed_http_only', + protocol: 'http', + domains: domainsByProto.http.map(d => formatDomainForSquid(d)), + description: 'Allow HTTP-only traffic to these domains (no HTTPS)', + }); + } + if (patternsByProto.http.length > 0) { + rules.push({ + id: 'allow-http-only-regex', + order: ++order, + action: 'allow', + aclName: 'allowed_http_only_regex', + protocol: 'http', + domains: patternsByProto.http.map(p => p.regex), + description: 'Allow HTTP-only traffic matching these patterns', + }); + } + + if (domainsByProto.https.length > 0) { + rules.push({ + id: 'allow-https-only-plain', + order: ++order, + action: 'allow', + aclName: 'allowed_https_only', + protocol: 'https', + domains: domainsByProto.https.map(d => formatDomainForSquid(d)), + description: 'Allow HTTPS-only traffic to these domains (no HTTP)', + }); + } + if (patternsByProto.https.length > 0) { + rules.push({ + id: 'allow-https-only-regex', + order: ++order, + action: 'allow', + aclName: 'allowed_https_only_regex', + protocol: 'https', + domains: patternsByProto.https.map(p => p.regex), + description: 'Allow HTTPS-only traffic matching these patterns', + }); + } + + // --- Both-protocol allow (used in deny rule logic) --- + if (domainsByProto.both.length > 0) { + rules.push({ + id: 'allow-both-plain', + order: ++order, + action: 'allow', + aclName: 'allowed_domains', + protocol: 'both', + domains: domainsByProto.both.map(d => formatDomainForSquid(d)), + description: 'Allow HTTP and HTTPS traffic to these domains', + }); + } + if (patternsByProto.both.length > 0) { + rules.push({ + id: 'allow-both-regex', + order: ++order, + action: 'allow', + aclName: 'allowed_domains_regex', + protocol: 'both', + domains: patternsByProto.both.map(p => p.regex), + description: 'Allow HTTP and HTTPS traffic matching these patterns', + }); + } + + // --- Default deny (final rule) --- + rules.push({ + id: 'deny-default', + order: ++order, + action: 'deny', + aclName: 'all', + protocol: 'both', + domains: [], + description: 'Deny all traffic not matching any allow rule (default deny)', + }); + + return { + version: 1, + generatedAt: new Date().toISOString(), + rules, + dangerousPorts: DANGEROUS_PORTS, + dnsServers: dnsServers || ['8.8.8.8', '8.8.4.4'], + sslBumpEnabled: sslBump ?? false, + dlpEnabled: enableDlp ?? false, + hostAccessEnabled: enableHostAccess ?? false, + allowHostPorts: allowHostPorts ?? null, + }; +} diff --git a/src/types.ts b/src/types.ts index 68288d58a..bd41ab12e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -340,6 +340,30 @@ export interface WrapperConfig { */ proxyLogsDir?: string; + /** + * Directory for firewall audit artifacts (configs, policy manifest, iptables state) + * + * When specified, audit artifacts are written directly to this directory + * during execution. This is useful for CI/CD where you want a predictable + * path for artifact upload. + * + * When not specified, audit artifacts are written to ${workDir}/audit/ + * during runtime and moved to /tmp/awf-audit- after cleanup. + * + * Artifacts include: + * - squid.conf: The generated Squid proxy configuration + * - docker-compose.redacted.yml: Container orchestration config (secrets redacted) + * - policy-manifest.json: Structured description of all firewall rules + * - iptables-audit.txt: Captured iptables state from the agent container + * + * Can be set via: + * - CLI flag: `--audit-dir ` + * - Environment variable: `AWF_AUDIT_DIR` + * + * @example '/tmp/gh-aw/sandbox/firewall/audit' + */ + auditDir?: string; + /** * Enable access to host services via host.docker.internal * @@ -794,6 +818,56 @@ export interface SquidConfig { dnsServers?: string[]; } +/** + * A single firewall policy rule as evaluated by Squid's http_access directives. + * + * Rules are evaluated in order; the first matching rule determines the outcome. + */ +export interface PolicyRule { + /** Unique identifier for this rule (e.g., "deny-blocked-plain", "allow-both-plain") */ + id: string; + /** Evaluation order (1-based, matching http_access line order) */ + order: number; + /** Whether this rule allows or denies traffic */ + action: 'allow' | 'deny'; + /** Squid ACL name (e.g., "allowed_domains", "blocked_domains_regex") */ + aclName: string; + /** Protocol scope: 'http' (non-CONNECT), 'https' (CONNECT), or 'both' */ + protocol: 'http' | 'https' | 'both'; + /** Domain values in this ACL (plain domains or regex patterns) */ + domains: string[]; + /** Human-readable description of this rule */ + description: string; +} + +/** + * Structured representation of the firewall policy in effect for a run. + * + * Written to policy-manifest.json in the audit directory. Combined with + * Squid access logs, this enables deterministic "which rule matched?" + * analysis by replaying ACL evaluation order. + */ +export interface PolicyManifest { + /** Schema version for forward compatibility */ + version: 1; + /** ISO timestamp when this manifest was generated */ + generatedAt: string; + /** Ordered list of http_access rules (evaluated first-to-last) */ + rules: PolicyRule[]; + /** TCP ports blocked by iptables (not redirected to Squid) */ + dangerousPorts: number[]; + /** DNS servers configured for the agent */ + dnsServers: string[]; + /** Whether SSL Bump (HTTPS inspection) is enabled */ + sslBumpEnabled: boolean; + /** Whether DLP scanning is enabled */ + dlpEnabled: boolean; + /** Whether host access is enabled */ + hostAccessEnabled: boolean; + /** Additional allowed ports (from --allow-host-ports), if any */ + allowHostPorts: string | null; +} + /** * Docker Compose configuration structure * From 11240d1c38fbd1ba062b9f247fc16ff7affc2bf2 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Mon, 23 Mar 2026 18:32:22 +0000 Subject: [PATCH 2/4] fix: address Copilot review comments and improve coverage - Fix IPv6 dest parsing in JSONL parser (use lastIndexOf for port) - Fix rule matcher to respect entry.isAllowed and skip empty-domain rules - Expand redaction patterns to cover _PAT and _AUTH env vars - Use DANGEROUS_PORTS array for iptables LOG rule (avoid drift) - Update docs: clarify manifest is higher-level policy, not literal directives - Add 6 stats-formatter tests for byRule output in all formats Co-Authored-By: Claude Opus 4.6 (1M context) --- containers/agent/setup-iptables.sh | 4 ++- src/docker-manager.ts | 5 ++- src/logs/audit-enricher.ts | 31 +++++++++++-------- src/logs/log-parser.ts | 40 +++++++++++++++++++++--- src/logs/stats-formatter.test.ts | 49 ++++++++++++++++++++++++++++++ src/squid-config.ts | 13 +++++--- src/types.ts | 10 ++++-- 7 files changed, 127 insertions(+), 25 deletions(-) diff --git a/containers/agent/setup-iptables.sh b/containers/agent/setup-iptables.sh index f54f517a8..e8efa0a45 100644 --- a/containers/agent/setup-iptables.sh +++ b/containers/agent/setup-iptables.sh @@ -316,7 +316,9 @@ fi # These ports are blocked by NAT RETURN + final DROP, but logging helps identify # what the agent tried to access echo "[iptables] Adding audit LOG rules for dangerous ports and default deny..." -iptables -A OUTPUT -p tcp -m multiport --dports 22,23,25,110,143,445,1433,1521,3306,3389,5432,6379,27017,27018,28017 \ +# Build comma-separated list from the DANGEROUS_PORTS array to stay in sync +DANGEROUS_PORTS_LIST="$(IFS=,; echo "${DANGEROUS_PORTS[*]}")" +iptables -A OUTPUT -p tcp -m multiport --dports "$DANGEROUS_PORTS_LIST" \ -m limit --limit 5/min --limit-burst 10 -j LOG --log-prefix "[FW_BLOCKED_DANGEROUS_PORT] " --log-level 4 --log-uid # Drop all other TCP and UDP traffic (default deny policy) diff --git a/src/docker-manager.ts b/src/docker-manager.ts index c8406e4f1..5eed00830 100644 --- a/src/docker-manager.ts +++ b/src/docker-manager.ts @@ -1438,7 +1438,10 @@ export function generateDockerCompose( * Replaces values of env vars that look like secrets (tokens, keys, passwords) with "[REDACTED]". */ function redactDockerComposeSecrets(compose: DockerComposeConfig): DockerComposeConfig { - const sensitivePatterns = /(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API_KEY|_B64)$/i; + // Match env var names ending with sensitive suffixes, or known token patterns. + // Covers: *_KEY, *_TOKEN, *_SECRET, *_PASSWORD, *_CREDENTIAL, *_B64, + // plus GITHUB_PAT (used in AWF_ONE_SHOT_TOKENS) and *_AUTH patterns. + const sensitivePatterns = /(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|_B64|_PAT|_AUTH)$/i; const redacted = JSON.parse(JSON.stringify(compose)) as DockerComposeConfig; for (const service of Object.values(redacted.services)) { diff --git a/src/logs/audit-enricher.ts b/src/logs/audit-enricher.ts index 31bc59ac3..0ee5a9b5b 100644 --- a/src/logs/audit-enricher.ts +++ b/src/logs/audit-enricher.ts @@ -85,25 +85,32 @@ function protocolMatches(rule: PolicyRule, isHttps: boolean): boolean { /** * Finds the first matching policy rule for a log entry by replaying * the http_access evaluation order. + * + * Only returns rules whose action is consistent with the observed decision + * (entry.isAllowed). Rules with empty domains (port/method-based rules like + * deny-unsafe-ports) cannot be deterministically replayed from log data alone, + * so they are skipped — the caller treats unmatched entries as "unknown". */ function findMatchingRule(entry: ParsedLogEntry, rules: PolicyRule[]): PolicyRule | null { + const expectedAction: 'allow' | 'deny' = entry.isAllowed ? 'allow' : 'deny'; + for (const rule of rules) { if (!protocolMatches(rule, entry.isHttps)) continue; - // The default deny rule (aclName: "all") matches everything - if (rule.aclName === 'all') return rule; - - // For deny rules with specific domains, match if domain is in the list - if (rule.action === 'deny' && domainMatchesRule(entry.domain, rule)) { - return rule; + // The default deny rule (aclName: "all") matches everything denied + if (rule.aclName === 'all') { + if (expectedAction === 'deny') return rule; + continue; } - // For allow rules, the Squid logic is: - // "http_access deny !allowed_domains" means: deny if NOT in allowed_domains - // So an allow rule matches if the domain IS in the list - if (rule.action === 'allow' && domainMatchesRule(entry.domain, rule)) { - return rule; - } + // Rules with no domains (port safety, DLP, etc.) can't be replayed + // from log data alone — skip to avoid misleading attribution + if (!rule.domains || rule.domains.length === 0) continue; + + if (!domainMatchesRule(entry.domain, rule)) continue; + + // Only attribute if the rule's action matches the observed outcome + if (rule.action === expectedAction) return rule; } return null; diff --git a/src/logs/log-parser.ts b/src/logs/log-parser.ts index a3affe0fe..701b7bc66 100644 --- a/src/logs/log-parser.ts +++ b/src/logs/log-parser.ts @@ -178,11 +178,41 @@ export function parseAuditJsonlLine(line: string): ParsedLogEntry | null { const decision = obj.decision || ''; const isAllowed = decision.startsWith('TCP_TUNNEL') || decision.startsWith('TCP_MISS'); - // Parse dest into IP and port - const destStr = obj.dest || '-:-'; - const destParts = destStr.split(':'); - const destIp = destParts[0] || '-'; - const destPort = destParts[1] || '-'; + // Parse dest into IP and port (handle IPv4, IPv6, and bracketed IPv6) + const rawDest = typeof obj.dest === 'string' ? obj.dest : ''; + let destIp = '-'; + let destPort = '-'; + + if (rawDest && rawDest !== '-:-') { + if (rawDest.startsWith('[')) { + // Bracketed IPv6, e.g. [2001:db8::1]:443 + const closeBracket = rawDest.indexOf(']'); + if (closeBracket !== -1) { + destIp = rawDest.slice(1, closeBracket) || '-'; + const remainder = rawDest.slice(closeBracket + 1); + if (remainder.startsWith(':')) { + const portCandidate = remainder.slice(1); + if (/^\d+$/.test(portCandidate)) destPort = portCandidate; + } + } else { + destIp = rawDest; + } + } else { + // Use last colon as port separator (safe for IPv4 ip:port) + const lastColon = rawDest.lastIndexOf(':'); + if (lastColon === -1) { + destIp = rawDest; + } else { + const portCandidate = rawDest.slice(lastColon + 1); + if (/^\d+$/.test(portCandidate)) { + destIp = rawDest.slice(0, lastColon) || '-'; + destPort = portCandidate; + } else { + destIp = rawDest; + } + } + } + } // Extract domain const url = obj.url || ''; diff --git a/src/logs/stats-formatter.test.ts b/src/logs/stats-formatter.test.ts index cfdd314da..f2adb48fa 100644 --- a/src/logs/stats-formatter.test.ts +++ b/src/logs/stats-formatter.test.ts @@ -9,6 +9,7 @@ import { formatStats, } from './stats-formatter'; import { AggregatedStats, DomainStats } from './log-aggregator'; +import { RuleStats } from './audit-enricher'; describe('stats-formatter', () => { describe('formatStatsJson', () => { @@ -245,3 +246,51 @@ function createSampleStats(): AggregatedStats { timeRange: { start: 1000, end: 2000 }, }; } + +describe('byRule stats in formatters', () => { + const ruleStats: RuleStats[] = [ + { ruleId: 'allow-both-plain', description: 'Allow domains', action: 'allow', hits: 8 }, + { ruleId: 'deny-default', description: 'Default deny', action: 'deny', hits: 2 }, + ]; + + function statsWithRules(): AggregatedStats { + return { ...createSampleStats(), byRule: ruleStats }; + } + + it('should include byRule in JSON output', () => { + const output = formatStatsJson(statsWithRules()); + const parsed = JSON.parse(output); + expect(parsed.byRule).toBeDefined(); + expect(parsed.byRule).toHaveLength(2); + expect(parsed.byRule[0].ruleId).toBe('allow-both-plain'); + }); + + it('should not include byRule in JSON when absent', () => { + const output = formatStatsJson(createSampleStats()); + const parsed = JSON.parse(output); + expect(parsed.byRule).toBeUndefined(); + }); + + it('should include Policy Rules section in markdown', () => { + const output = formatStatsMarkdown(statsWithRules()); + expect(output).toContain('Policy Rules'); + expect(output).toContain('allow-both-plain'); + expect(output).toContain('deny-default'); + }); + + it('should not include Policy Rules in markdown when absent', () => { + const output = formatStatsMarkdown(createSampleStats()); + expect(output).not.toContain('Policy Rules'); + }); + + it('should include Policy Rules in pretty output', () => { + const output = formatStatsPretty(statsWithRules(), false); + expect(output).toContain('Policy Rules'); + expect(output).toContain('allow-both-plain'); + }); + + it('should not include Policy Rules in pretty when absent', () => { + const output = formatStatsPretty(createSampleStats(), false); + expect(output).not.toContain('Policy Rules'); + }); +}); diff --git a/src/squid-config.ts b/src/squid-config.ts index ba540d4a3..73deae6a8 100644 --- a/src/squid-config.ts +++ b/src/squid-config.ts @@ -630,11 +630,16 @@ shutdown_lifetime 0 seconds } /** - * Generates a structured policy manifest describing all firewall rules in effect. + * Generates a structured policy manifest describing all effective access-control rules. * - * The manifest mirrors the http_access rule ordering in generateSquidConfig() so - * that post-hoc log analysis can deterministically replay which rule matched each - * request by evaluating rules in order and stopping at the first match. + * The manifest reflects the logical policy and overall evaluation order derived from + * generateSquidConfig(), but it is a higher-level representation rather than a literal + * list of Squid `http_access` directives. Some internal rules (negations, method + * constraints, localhost/localnet allowances) are abstracted into logical concepts. + * + * Port/method-based rules (deny-unsafe-ports, deny-dlp) have empty `domains` arrays + * because they can't be deterministically replayed from Squid log data alone — the + * enricher skips them and attributes those denials to "unknown". */ export function generatePolicyManifest(config: SquidConfig): PolicyManifest { const { domains, blockedDomains, sslBump, enableHostAccess, allowHostPorts, enableDlp, dnsServers } = config; diff --git a/src/types.ts b/src/types.ts index bd41ab12e..fdf361a5a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -830,7 +830,7 @@ export interface PolicyRule { order: number; /** Whether this rule allows or denies traffic */ action: 'allow' | 'deny'; - /** Squid ACL name (e.g., "allowed_domains", "blocked_domains_regex") */ + /** Squid ACL name or expression (e.g., "allowed_domains", "!Safe_ports", "dst_ipv4"). Not always a single ACL name — may include negation or method constraints for port/method-based rules. */ aclName: string; /** Protocol scope: 'http' (non-CONNECT), 'https' (CONNECT), or 'both' */ protocol: 'http' | 'https' | 'both'; @@ -854,7 +854,13 @@ export interface PolicyManifest { generatedAt: string; /** Ordered list of http_access rules (evaluated first-to-last) */ rules: PolicyRule[]; - /** TCP ports blocked by iptables (not redirected to Squid) */ + /** + * TCP ports treated as "dangerous" by the firewall policy. + * + * Derived from the Squid configuration (DANGEROUS_PORTS) used by the wrapper. + * Documents which ports are considered unsafe for direct proxying. May not be + * an exact reflection of the iptables rules installed in the agent container. + */ dangerousPorts: number[]; /** DNS servers configured for the agent */ dnsServers: string[]; From 1bdad8d47a8abda70024bfabf3cd19b7dc10695d Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Mon, 23 Mar 2026 18:37:21 +0000 Subject: [PATCH 3/4] docs: add real audit artifact samples from local awf run Samples generated from: sudo awf --allow-domains github.com,api.github.com --build-local -- bash -c 'curl -s https://api.github.com/zen; curl https://evil.example.com || true' Includes: access.log, audit.jsonl, policy-manifest.json, squid.conf, docker-compose.redacted.yml Co-Authored-By: Claude Opus 4.6 (1M context) --- samples/audit/README.md | 28 ++++++ samples/audit/access.log | 3 + samples/audit/audit.jsonl | 3 + samples/audit/docker-compose.redacted.yml | 103 ++++++++++++++++++++++ samples/audit/policy-manifest.json | 76 ++++++++++++++++ samples/audit/squid.conf | 97 ++++++++++++++++++++ 6 files changed, 310 insertions(+) create mode 100644 samples/audit/README.md create mode 100644 samples/audit/access.log create mode 100644 samples/audit/audit.jsonl create mode 100644 samples/audit/docker-compose.redacted.yml create mode 100644 samples/audit/policy-manifest.json create mode 100644 samples/audit/squid.conf diff --git a/samples/audit/README.md b/samples/audit/README.md new file mode 100644 index 000000000..ebbca3bc0 --- /dev/null +++ b/samples/audit/README.md @@ -0,0 +1,28 @@ +# Audit Artifact Samples + +These are **real** audit artifacts generated by running `awf` locally: + +```bash +sudo awf --allow-domains github.com,api.github.com \ + --audit-dir /tmp/audit-sample \ + --build-local \ + -- bash -c 'curl -s https://api.github.com/zen; curl -s https://evil.example.com || true; sleep 2' +``` + +## Files + +| File | Description | +|------|-------------| +| `policy-manifest.json` | Structured description of all firewall rules with evaluation order | +| `access.log` | Squid access log in the `firewall_detailed` text format | +| `audit.jsonl` | Squid access log in structured JSONL format (machine-readable) | +| `squid.conf` | Generated Squid proxy configuration snapshot | +| `docker-compose.redacted.yml` | Container orchestration config with secrets replaced by `[REDACTED]` | + +## What to look for + +- In `access.log`: `TCP_TUNNEL:HIER_DIRECT` = allowed, `TCP_DENIED:HIER_NONE` = blocked +- In `audit.jsonl`: Same data in JSON format, one object per line +- In `policy-manifest.json`: Rules evaluated top-to-bottom; `deny-unsafe-ports` and `deny-raw-ipv4` come before domain rules +- In `squid.conf`: The actual ACL rules and log format directives +- In `docker-compose.redacted.yml`: Note `AWF_SQUID_CONFIG_B64: '[REDACTED]'` — secrets are stripped diff --git a/samples/audit/access.log b/samples/audit/access.log new file mode 100644 index 000000000..52a568c2a --- /dev/null +++ b/samples/audit/access.log @@ -0,0 +1,3 @@ +1774290908.910 172.30.0.20:55872 api.github.com:443 140.82.116.5:443 1.1 CONNECT 200 TCP_TUNNEL:HIER_DIRECT api.github.com:443 "curl/7.81.0" +1774290909.180 172.30.0.20:55880 api.github.com:443 140.82.116.5:443 1.1 CONNECT 200 TCP_TUNNEL:HIER_DIRECT api.github.com:443 "curl/7.81.0" +1774290909.186 172.30.0.20:55890 evil.example.com:443 -:- 1.1 CONNECT 403 TCP_DENIED:HIER_NONE evil.example.com:443 "curl/7.81.0" diff --git a/samples/audit/audit.jsonl b/samples/audit/audit.jsonl new file mode 100644 index 000000000..b3adce217 --- /dev/null +++ b/samples/audit/audit.jsonl @@ -0,0 +1,3 @@ +{"ts":1774290908.910,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} +{"ts":1774290909.180,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} +{"ts":1774290909.186,"client":"172.30.0.20","host":"evil.example.com:443","dest":"-:-","method":"CONNECT","status":403,"decision":"TCP_DENIED","url":"evil.example.com:443"} diff --git a/samples/audit/docker-compose.redacted.yml b/samples/audit/docker-compose.redacted.yml new file mode 100644 index 000000000..d97546c8b --- /dev/null +++ b/samples/audit/docker-compose.redacted.yml @@ -0,0 +1,103 @@ +services: + squid-proxy: + container_name: awf-squid + networks: + awf-net: + ipv4_address: 172.30.0.10 + volumes: + - /tmp/awf-1774290893689/squid-logs:/var/log/squid:rw + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "3128"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + ports: + - '3128:3128' + cap_drop: + - NET_RAW + - SYS_ADMIN + - SYS_PTRACE + - SYS_MODULE + - MKNOD + - AUDIT_WRITE + - SETFCAP + stop_grace_period: 2s + environment: + AWF_SQUID_CONFIG_B64: '[REDACTED]' + entrypoint: + - /bin/bash + - '-c' + - echo "$$AWF_SQUID_CONFIG_B64" | base64 -d > /etc/squid/squid.conf && exec /usr/local/bin/entrypoint.sh + build: + context: ./containers/squid + dockerfile: Dockerfile + agent: + container_name: awf-agent + networks: + awf-net: + ipv4_address: 172.30.0.20 + dns: [8.8.8.8, 8.8.4.4] + dns_search: [] + volumes: + - /tmp:/tmp:rw + - $WORKSPACE:$WORKSPACE:rw + - /tmp/awf-TIMESTAMP/agent-logs:$HOME/.copilot/logs:rw + - /tmp/awf-TIMESTAMP/init-signal:/tmp/awf-init:rw + - /usr:/host/usr:ro + - /bin:/host/bin:ro + - /sbin:/host/sbin:ro + - /lib:/host/lib:ro + - /lib64:/host/lib64:ro + # ... (selective bind mounts for system binaries, home dirs, etc.) + environment: + HTTP_PROXY: http://172.30.0.10:3128 + HTTPS_PROXY: http://172.30.0.10:3128 + https_proxy: http://172.30.0.10:3128 + SQUID_PROXY_HOST: squid-proxy + SQUID_PROXY_PORT: '3128' + HOME: /home/runner + NO_COLOR: '1' + AWF_ONE_SHOT_TOKENS: COPILOT_GITHUB_TOKEN,GITHUB_TOKEN,GH_TOKEN,... + AWF_DNS_SERVERS: 8.8.8.8,8.8.4.4 + AWF_CHROOT_ENABLED: 'true' + AWF_WORKDIR: /home/runner + AWF_USER_UID: '1000' + AWF_USER_GID: '1000' + depends_on: + squid-proxy: + condition: service_healthy + cap_add: [SYS_CHROOT, SYS_ADMIN] + cap_drop: [NET_RAW, SYS_PTRACE, SYS_MODULE, SYS_RAWIO, MKNOD] + security_opt: + - no-new-privileges:true + - seccomp=/tmp/awf-TIMESTAMP/seccomp-profile.json + - apparmor:unconfined + mem_limit: 6g + pids_limit: 1000 + tty: false + command: + - /bin/bash + - '-c' + - 'curl -s https://api.github.com/zen; curl -s https://evil.example.com || true; sleep 2' + iptables-init: + container_name: awf-iptables-init + network_mode: service:agent + volumes: + - /tmp/awf-TIMESTAMP/init-signal:/tmp/awf-init:rw + environment: + SQUID_PROXY_HOST: 172.30.0.10 + SQUID_PROXY_PORT: '3128' + AWF_DNS_SERVERS: 8.8.8.8,8.8.4.4 + depends_on: + agent: + condition: service_healthy + cap_add: [NET_ADMIN, NET_RAW] + cap_drop: [ALL] + entrypoint: [/bin/bash] + command: ['-c', '/usr/local/bin/setup-iptables.sh > /tmp/awf-init/output.log 2>&1 && touch /tmp/awf-init/ready'] + mem_limit: 128m + restart: 'no' +networks: + awf-net: + external: true diff --git a/samples/audit/policy-manifest.json b/samples/audit/policy-manifest.json new file mode 100644 index 000000000..e0965108f --- /dev/null +++ b/samples/audit/policy-manifest.json @@ -0,0 +1,76 @@ +{ + "version": 1, + "generatedAt": "2026-03-23T18:34:53.894Z", + "rules": [ + { + "id": "deny-unsafe-ports", + "order": 1, + "action": "deny", + "aclName": "!Safe_ports", + "protocol": "both", + "domains": [], + "description": "Deny requests to ports not in Safe_ports ACL (only 80, 443, and user-specified ports allowed)" + }, + { + "id": "deny-connect-unsafe-ports", + "order": 2, + "action": "deny", + "aclName": "CONNECT !Safe_ports", + "protocol": "https", + "domains": [], + "description": "Deny CONNECT (HTTPS) to ports not in Safe_ports ACL" + }, + { + "id": "deny-raw-ipv4", + "order": 3, + "action": "deny", + "aclName": "dst_ipv4", + "protocol": "both", + "domains": [ + "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$" + ], + "description": "Deny requests to raw IPv4 addresses (bypasses domain filtering)" + }, + { + "id": "deny-raw-ipv6", + "order": 4, + "action": "deny", + "aclName": "dst_ipv6", + "protocol": "both", + "domains": [ + "^\\[?[0-9a-fA-F:]+\\]?$" + ], + "description": "Deny requests to raw IPv6 addresses (bypasses domain filtering)" + }, + { + "id": "allow-both-plain", + "order": 5, + "action": "allow", + "aclName": "allowed_domains", + "protocol": "both", + "domains": [ + ".github.com" + ], + "description": "Allow HTTP and HTTPS traffic to these domains" + }, + { + "id": "deny-default", + "order": 6, + "action": "deny", + "aclName": "all", + "protocol": "both", + "domains": [], + "description": "Deny all traffic not matching any allow rule (default deny)" + } + ], + "dangerousPorts": [ + 22, 23, 25, 110, 143, 445, 1433, 1521, 3306, 3389, + 5432, 5984, 6379, 6984, 8086, 8088, 9200, 9300, + 27017, 27018, 28017 + ], + "dnsServers": ["8.8.8.8", "8.8.4.4"], + "sslBumpEnabled": false, + "dlpEnabled": false, + "hostAccessEnabled": false, + "allowHostPorts": null +} diff --git a/samples/audit/squid.conf b/samples/audit/squid.conf new file mode 100644 index 000000000..42551a9d7 --- /dev/null +++ b/samples/audit/squid.conf @@ -0,0 +1,97 @@ +# Squid configuration for egress traffic control +# Generated by awf + + +# Disable pinger (ICMP) - requires NET_RAW capability which we don't have for security +pinger_enable off + +# PID file location - use proxy-owned directory since container runs as non-root +pid_filename /var/run/squid/squid.pid + +# Custom log format with detailed connection information +# Format: timestamp client_ip:port dest_domain dest_ip:port protocol method status decision url user_agent +# Note: For CONNECT requests (HTTPS), the domain is in the URL field +logformat firewall_detailed %ts.%03tu %>a:%>p %{Host}>h %Hs %Ss:%Sh %ru "%{User-Agent}>h" + +# Structured JSONL audit log for machine-readable analysis +# Note: Squid logformat does not JSON-escape strings, so fields like User-Agent +# could break JSON parsing. We omit User-Agent to reduce breakage risk. +logformat audit_jsonl {"ts":%ts.%03tu,"client":"%>a","host":"%{Host}>h","dest":"%Hs,"decision":"%Ss","url":"%ru"} + +# Access log and cache configuration +# Don't log healthcheck probes from localhost (using ACL filter on access_log) +acl healthcheck_localhost src 127.0.0.1 ::1 +access_log /var/log/squid/access.log firewall_detailed !healthcheck_localhost +access_log /var/log/squid/audit.jsonl audit_jsonl !healthcheck_localhost +cache_log /var/log/squid/cache.log +cache deny all + +# ACL definitions for allowed domains (HTTP and HTTPS) +acl allowed_domains dstdomain .github.com + +# Port configuration +http_port 3128 + + +# Network ACLs +acl localnet src 10.0.0.0/8 +acl localnet src 172.16.0.0/12 +acl localnet src 192.168.0.0/16 +acl localnet src fc00::/7 +acl localnet src fe80::/10 + +# Port ACLs +acl SSL_ports port 443 +acl Safe_ports port 80 # HTTP +acl Safe_ports port 443 # HTTPS +acl CONNECT method CONNECT + +# Access rules +# Deny unsafe ports (only allow Safe_ports defined above) +http_access deny !Safe_ports +# Allow CONNECT to Safe_ports instead of just SSL_ports (443) +http_access deny CONNECT !Safe_ports + +# Deny CONNECT to raw IP addresses (IPv4 and IPv6) +# Prevents bypassing domain-based filtering via direct IP connections +acl dst_ipv4 dstdom_regex ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ +acl dst_ipv6 dstdom_regex ^\[?[0-9a-fA-F:]+\]?$ +http_access deny dst_ipv4 +http_access deny dst_ipv6 + +# Deny requests to unknown domains (not in allow-list) +http_access deny !allowed_domains + +# Allow from trusted sources (after domain filtering) +http_access allow localnet +http_access allow localhost + +# Deny everything else +http_access deny all + +# Disable caching +cache deny all + +# DNS settings - Squid resolves all domains for HTTP/HTTPS traffic +dns_nameservers 8.8.8.8 8.8.4.4 + +# Forwarded headers +forwarded_for delete +via off + +# Error page customization +error_directory /usr/share/squid/errors/en + +# Memory and file descriptor limits +cache_mem 64 MB +maximum_object_size 0 KB + +# Timeout settings for streaming/long-lived connections (AI inference APIs) +read_timeout 30 minutes +connect_timeout 30 seconds +request_timeout 2 minutes +persistent_request_timeout 2 minutes +pconn_timeout 2 minutes +client_lifetime 8 hours +half_closed_clients on +shutdown_lifetime 0 seconds From 847e6529bcf730e72a3a761e3bcb3e957a4553d6 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Mon, 23 Mar 2026 19:00:13 +0000 Subject: [PATCH 4/4] fix: address second round of Copilot review comments - Expand redaction to substring match (catches CREDENTIALS, PRIVATE_KEY) - Use restrictive perms (0o600/0o700) for audit artifacts initially - Output valid JSON array from `awf logs audit --format json` (not JSONL) - Include "unknown" row in computeRuleStats for unattributed traffic - Demote AWF_AUDIT_DIR to fallback in manifest discovery (avoid cross-run mismatch) - Broaden isAllowed: treat non-TCP_DENIED/NONE as allowed (covers TCP_HIT etc.) - Update tests for new isAllowed semantics Co-Authored-By: Claude Opus 4.6 (1M context) --- src/commands/logs-audit.ts | 5 +++-- src/commands/logs-command-helpers.ts | 4 +++- src/docker-manager.ts | 17 +++++++++-------- src/logs/audit-enricher.ts | 15 ++++++++++++++- src/logs/log-parser.test.ts | 11 +++++------ src/logs/log-parser.ts | 15 ++++++++++++--- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/commands/logs-audit.ts b/src/commands/logs-audit.ts index ad89423dd..45aa5f7dd 100644 --- a/src/commands/logs-audit.ts +++ b/src/commands/logs-audit.ts @@ -28,7 +28,7 @@ export interface AuditCommandOptions { } function formatAuditJson(entries: EnrichedLogEntry[]): string { - return entries.map(e => JSON.stringify({ + const items = entries.map(e => ({ timestamp: e.timestamp, domain: e.domain, method: e.method, @@ -37,7 +37,8 @@ function formatAuditJson(entries: EnrichedLogEntry[]): string { matchedRule: e.matchedRuleId, matchReason: e.matchReason, url: e.url, - })).join('\n'); + })); + return JSON.stringify(items, null, 2); } function formatAuditMarkdown(entries: EnrichedLogEntry[], manifest: PolicyManifest): string { diff --git a/src/commands/logs-command-helpers.ts b/src/commands/logs-command-helpers.ts index f96c120bf..73b2624c6 100644 --- a/src/commands/logs-command-helpers.ts +++ b/src/commands/logs-command-helpers.ts @@ -95,9 +95,11 @@ export function findPolicyManifestForSource(source: LogSource): PolicyManifest | source.path.replace(/squid-logs-/, 'awf-audit-').replace(/\/?$/, '/policy-manifest.json'), ]; + // AWF_AUDIT_DIR is a fallback, not priority — prefer manifests co-located with + // the selected log source to avoid cross-run mismatch const auditDirEnv = process.env.AWF_AUDIT_DIR; if (auditDirEnv) { - candidates.unshift(path.join(auditDirEnv, 'policy-manifest.json')); + candidates.push(path.join(auditDirEnv, 'policy-manifest.json')); } for (const candidate of candidates) { diff --git a/src/docker-manager.ts b/src/docker-manager.ts index 5eed00830..8155d76f9 100644 --- a/src/docker-manager.ts +++ b/src/docker-manager.ts @@ -1438,10 +1438,10 @@ export function generateDockerCompose( * Replaces values of env vars that look like secrets (tokens, keys, passwords) with "[REDACTED]". */ function redactDockerComposeSecrets(compose: DockerComposeConfig): DockerComposeConfig { - // Match env var names ending with sensitive suffixes, or known token patterns. - // Covers: *_KEY, *_TOKEN, *_SECRET, *_PASSWORD, *_CREDENTIAL, *_B64, - // plus GITHUB_PAT (used in AWF_ONE_SHOT_TOKENS) and *_AUTH patterns. - const sensitivePatterns = /(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|_B64|_PAT|_AUTH)$/i; + // Match env var names containing sensitive keywords. + // Uses substring matching (not just suffix) to catch patterns like + // GOOGLE_APPLICATION_CREDENTIALS, PRIVATE_KEY_PATH, etc. + const sensitivePatterns = /(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?|_B64|_PAT|_AUTH|PRIVATE_KEY)/i; const redacted = JSON.parse(JSON.stringify(compose)) as DockerComposeConfig; for (const service of Object.values(redacted.services)) { @@ -1654,18 +1654,19 @@ export async function writeConfigs(config: WrapperConfig): Promise { // Write audit artifacts (config snapshots for post-run forensics) const auditDir = config.auditDir || path.join(config.workDir, 'audit'); if (!fs.existsSync(auditDir)) { - fs.mkdirSync(auditDir, { recursive: true, mode: 0o755 }); + // Restrictive permissions initially; made readable during cleanup (chmod a+rX) + fs.mkdirSync(auditDir, { recursive: true, mode: 0o700 }); } // Save squid.conf for audit (no secrets — just domain ACLs and proxy config) - fs.writeFileSync(path.join(auditDir, 'squid.conf'), squidConfig, { mode: 0o644 }); + fs.writeFileSync(path.join(auditDir, 'squid.conf'), squidConfig, { mode: 0o600 }); // Save redacted docker-compose.yml (strip env vars that may contain secrets) const redactedCompose = redactDockerComposeSecrets(dockerCompose); fs.writeFileSync( path.join(auditDir, 'docker-compose.redacted.yml'), yaml.dump(redactedCompose, { lineWidth: -1 }), - { mode: 0o644 } + { mode: 0o600 } ); // Generate and save policy manifest (structured description of all firewall rules) @@ -1682,7 +1683,7 @@ export async function writeConfigs(config: WrapperConfig): Promise { fs.writeFileSync( path.join(auditDir, 'policy-manifest.json'), JSON.stringify(policyManifest, null, 2), - { mode: 0o644 } + { mode: 0o600 } ); logger.debug(`Audit artifacts written to: ${auditDir}`); diff --git a/src/logs/audit-enricher.ts b/src/logs/audit-enricher.ts index 0ee5a9b5b..6ce089521 100644 --- a/src/logs/audit-enricher.ts +++ b/src/logs/audit-enricher.ts @@ -164,10 +164,23 @@ export function computeRuleStats( hitCounts.set(entry.matchedRuleId, (hitCounts.get(entry.matchedRuleId) || 0) + 1); } - return manifest.rules.map(rule => ({ + const manifestStats: RuleStats[] = manifest.rules.map(rule => ({ ruleId: rule.id, description: rule.description, action: rule.action, hits: hitCounts.get(rule.id) || 0, })); + + // Include unattributed traffic so per-rule totals reconcile with overall totals + const unknownHits = hitCounts.get('unknown') || 0; + if (unknownHits > 0) { + manifestStats.push({ + ruleId: 'unknown', + description: 'Unattributed traffic (port/method-based rules not replayable from logs)', + action: 'deny', + hits: unknownHits, + }); + } + + return manifestStats; } diff --git a/src/logs/log-parser.test.ts b/src/logs/log-parser.test.ts index 75ddc2993..e497aa8a8 100644 --- a/src/logs/log-parser.test.ts +++ b/src/logs/log-parser.test.ts @@ -103,19 +103,18 @@ describe('log-parser', () => { const result = parseLogLine(line); expect(result).not.toBeNull(); - // TCP_HIT is neither TCP_TUNNEL nor TCP_MISS, so isAllowed should be false - // This documents current behavior: only TCP_TUNNEL and TCP_MISS are considered allowed - expect(result!.isAllowed).toBe(false); + // TCP_HIT is a successful cache hit — should be treated as allowed + expect(result!.isAllowed).toBe(true); }); - it('should mark TCP_REFRESH_MODIFIED as denied (not in allowed list)', () => { + it('should mark TCP_REFRESH_MODIFIED as allowed (refreshed cache)', () => { const line = '1761074374.646 172.30.0.20:39748 example.com:80 93.184.216.34:80 1.1 GET 200 TCP_REFRESH_MODIFIED:HIER_DIRECT http://example.com/ "-"'; const result = parseLogLine(line); expect(result).not.toBeNull(); - // TCP_REFRESH_MODIFIED is not TCP_TUNNEL or TCP_MISS - expect(result!.isAllowed).toBe(false); + // TCP_REFRESH_MODIFIED is a successful refreshed response — should be allowed + expect(result!.isAllowed).toBe(true); }); it('should mark NONE_NONE as denied (connection failure entries)', () => { diff --git a/src/logs/log-parser.ts b/src/logs/log-parser.ts index 701b7bc66..dbf222df1 100644 --- a/src/logs/log-parser.ts +++ b/src/logs/log-parser.ts @@ -65,7 +65,11 @@ export function parseLogLine(line: string): ParsedLogEntry | null { const timestamp = parseFloat(timestampStr); const statusCode = parseInt(statusCodeStr, 10); - const isAllowed = decision.startsWith('TCP_TUNNEL') || decision.startsWith('TCP_MISS'); + // Treat anything not explicitly denied (TCP_DENIED) or noise (NONE) as allowed. + // Covers TCP_TUNNEL, TCP_MISS, TCP_HIT, TCP_MEM_HIT, TCP_REFRESH_*, etc. + const isDenied = decision.startsWith('TCP_DENIED'); + const isNone = decision.startsWith('NONE'); + const isAllowed = decision !== '' && !isDenied && !isNone; const isHttps = method === 'CONNECT'; // Extract domain from the appropriate field @@ -175,8 +179,13 @@ export function parseAuditJsonlLine(line: string): ParsedLogEntry | null { const method = obj.method || ''; const isHttps = method === 'CONNECT'; - const decision = obj.decision || ''; - const isAllowed = decision.startsWith('TCP_TUNNEL') || decision.startsWith('TCP_MISS'); + // Squid can emit many TCP_* statuses for allowed requests (TCP_TUNNEL, TCP_MISS, + // TCP_HIT, TCP_MEM_HIT, TCP_REFRESH_HIT, etc.). Treat anything that is not + // explicitly denied (TCP_DENIED) or operational noise (NONE) as allowed. + const decision = typeof obj.decision === 'string' ? obj.decision : ''; + const isDenied = decision.startsWith('TCP_DENIED'); + const isNone = decision.startsWith('NONE'); + const isAllowed = decision !== '' && !isDenied && !isNone; // Parse dest into IP and port (handle IPv4, IPv6, and bracketed IPv6) const rawDest = typeof obj.dest === 'string' ? obj.dest : '';