diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py index 93a8b50cf9..be85faef55 100644 --- a/bbot/core/event/helpers.py +++ b/bbot/core/event/helpers.py @@ -111,7 +111,7 @@ def _sanitize_and_extract_host(self, data): """ return data, None, None - async def _generate_children(self, ssl_verify=False): + async def _generate_children(self, helpers): return [] def _override_input(self, input): @@ -290,23 +290,15 @@ class ASN(BaseEventSeed): def _override_input(self, input): return f"ASN:{self.data}" - # ASNs are essentially just a superset of IP_RANGES. - # This method resolves the ASN to a list of IP_RANGES using the ASN API, and then adds the cidr string as a child event seed. - # These will later be automatically resolved to an IP_RANGE event seed and added to the target. - async def _generate_children(self, ssl_verify=False): - from asndb import ASNDB - - client = ASNDB(verify=ssl_verify) - asn_data = await client.lookup_asn(str(self.data), include_subnets=True) - children = [] - if asn_data: - subnets = asn_data.get("subnets") - if isinstance(subnets, str): - subnets = [subnets] - if subnets: - for cidr in subnets: - children.append(cidr) - return children + # ASNs are essentially a superset of IP_RANGES. This resolves the ASN to its + # subnets via the shared ASN helper and emits each CIDR as a child seed, + # which is later resolved to an IP_RANGE seed and added to the target. + async def _generate_children(self, helpers): + asn_data = await helpers.asn.asn_to_subnets(self.data) + subnets = asn_data.get("subnets") or [] + if isinstance(subnets, str): + subnets = [subnets] + return list(subnets) @staticmethod def handle_match(match): diff --git a/bbot/core/helpers/asn.py b/bbot/core/helpers/asn.py index 055970a647..5da042f19c 100644 --- a/bbot/core/helpers/asn.py +++ b/bbot/core/helpers/asn.py @@ -1,5 +1,7 @@ import logging +from bbot.errors import ASNResolutionError + log = logging.getLogger("bbot.core.helpers.asn") @@ -19,9 +21,15 @@ class ASNHelper: "country": "Unknown", } + FAILURE_THRESHOLD = 5 + MAX_RETRIES = 3 + RETRY_DELAY = 3 + def __init__(self, parent_helper): self.parent_helper = parent_helper self._client = None + self._consecutive_failures = 0 + self._circuit_broken = False @property def client(self): @@ -45,29 +53,50 @@ def _normalize(self, response): "country": response.get("country") or "", } + def _record_failure(self): + self._consecutive_failures += 1 + if self._consecutive_failures >= self.FAILURE_THRESHOLD and not self._circuit_broken: + self._circuit_broken = True + log.critical( + "ASN lookups disabled for the rest of this scan after %d consecutive failures. " + "The bbot.io ASN API could not be reached (this may be due to regional network restrictions). " + "ASN enrichment data will not be available.", + self.FAILURE_THRESHOLD, + ) + async def ip_to_subnets(self, ip): """Return ASN info for an IP address.""" + if self._circuit_broken: + return self.UNKNOWN_ASN try: response = await self.client.lookup_ip(str(ip), include_subnets=True) except Exception as e: log.warning(f"ASN lookup failed for IP {ip}: {e}") + self._record_failure() return self.UNKNOWN_ASN + self._consecutive_failures = 0 return self._normalize(response) async def asn_to_subnets(self, asn): - """Return ASN info (including subnets) for an ASN number.""" - if isinstance(asn, str): + """Resolve an ASN number to its subnets, retrying on transient failure. + + Used by ASN-as-target seed expansion, which cannot degrade gracefully: + without the ASN's subnets there is nothing to scan. Raises + ASNResolutionError if the ASN can't be resolved after MAX_RETRIES so the + scan aborts with guidance instead of silently scanning nothing. + """ + asn = int(str(asn).lower().lstrip("as")) + last_error = None + for attempt in range(1, self.MAX_RETRIES + 1): try: - asn = int(asn.lower().lstrip("as")) - except ValueError: - log.warning(f"Invalid ASN format: {asn}") - return self.UNKNOWN_ASN - try: - response = await self.client.lookup_asn(str(asn), include_subnets=True) - except Exception as e: - log.warning(f"ASN lookup failed for AS{asn}: {e}") - return self.UNKNOWN_ASN - return self._normalize(response) + response = await self.client.lookup_asn(asn, include_subnets=True) + return self._normalize(response) + except Exception as e: + last_error = e + log.warning(f"ASN resolution attempt {attempt}/{self.MAX_RETRIES} failed for AS{asn}: {e}") + if attempt < self.MAX_RETRIES: + await self.parent_helper.sleep(self.RETRY_DELAY) + raise ASNResolutionError(f"AS{asn}: {last_error}") async def cleanup(self): """Clean up the asndb client.""" diff --git a/bbot/errors.py b/bbot/errors.py index db295da81d..43f68b5f6f 100644 --- a/bbot/errors.py +++ b/bbot/errors.py @@ -76,3 +76,7 @@ class DNSError(BBOTEngineError): class ExcavateError(BBOTError): pass + + +class ASNResolutionError(BBOTError): + pass diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 54455e3bf0..87eef80414 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -18,8 +18,7 @@ class asn(BaseReportModule): async def setup(self): self.unknown_asn = ASNHelper.UNKNOWN_ASN - # Track ASN counts locally for reporting - self.asn_counts = {} # ASN number -> count mapping + self.asn_metadata = {} return True async def filter_event(self, event): @@ -41,10 +40,14 @@ async def handle_event(self, event): asn_country = asn_data.get("country", "") subnets = asn_data.get("subnets", []) - # Track ASN subnet counts for reporting (only once per ASN) if asn_number and asn_number != 0: - if asn_number not in self.asn_counts: - self.asn_counts[asn_number] = len(subnets) + if asn_number not in self.asn_metadata: + self.asn_metadata[asn_number] = { + "subnet_count": len(subnets), + "name": asn_name, + "description": asn_description, + "country": asn_country, + } # Don't emit ASN 0 - it's reserved and indicates unknown ASN data if asn_number != 0: @@ -56,34 +59,21 @@ async def handle_event(self, event): ) async def report(self): - """Generate an ASN summary table based on locally tracked ASN counts.""" - - if not self.asn_counts: + if not self.asn_metadata: return - # Build table rows sorted by ASN number (low to high) - sorted_asns = sorted(self.asn_counts.items(), key=lambda x: int(x[0])) + sorted_asns = sorted(self.asn_metadata.items(), key=lambda x: int(x[0])) header = ["ASN", "Subnet Count", "Name", "Description", "Country"] table = [] - for asn_number, subnet_count in sorted_asns: - # Get ASN details from helper - asn_data = await self.helpers.asn.asn_to_subnets(asn_number) - if asn_data: - asn_name = asn_data.get("name", "") - asn_description = asn_data.get("description", "") - asn_country = asn_data.get("country", "") - else: - asn_name = asn_description = asn_country = "unknown" - - number = f"AS{asn_number}" if asn_number != 0 else str(asn_number) + for asn_number, metadata in sorted_asns: table.append( [ - number, - f"{subnet_count:,}", - asn_name, - asn_description, - asn_country, + f"AS{asn_number}", + f"{metadata['subnet_count']:,}", + metadata["name"], + metadata["description"], + metadata["country"], ] ) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index a223e6751a..182d6e1beb 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -20,7 +20,7 @@ from bbot.core.multiprocess import SHARED_INTERPRETER_STATE from bbot.core.helpers.async_helpers import async_to_sync_gen from bbot.logger import log_to_stderr -from bbot.errors import BBOTError, ScanError, ValidationError +from bbot.errors import ASNResolutionError, BBOTError, ScanError, ValidationError from bbot.constants import ( get_scan_status_code, get_scan_status_name, @@ -327,9 +327,15 @@ async def _prep(self): Expands async seed types (e.g. ASN → IP ranges), evaluates preset conditions, creates the scan's output folder, loads its modules, and calls their .setup() methods. """ - # expand async seed types (e.g. ASN → IP ranges) - ssl_verify = self.preset.web_config.get("ssl_verify_infrastructure", True) - await self.preset.target.generate_children(ssl_verify=ssl_verify) + # expand async seed types (e.g. ASN -> IP ranges) + try: + await self.preset.target.generate_children(helpers=self.helpers) + except ASNResolutionError as e: + raise ScanError( + f"Failed to resolve ASN target ({e}). " + f"The bbot.io ASN API could not be reached; this may be due to regional network restrictions or a temporary outage. " + f"To scan this ASN's networks, look up its prefixes (e.g. at bgp.tools) and pass them directly: bbot -t 1.2.3.0/24 5.6.0.0/16" + ) # evaluate preset conditions (may abort the scan) if self.preset.conditions: diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index fb925eb4a4..78fcb144f4 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -410,7 +410,7 @@ def in_target(self, host): def __eq__(self, other): return self.hash == other.hash - async def generate_children(self, ssl_verify=False): + async def generate_children(self, helpers): """ Generate children for the target, for seed types that expand into other seed types. E.g. ASN targets are expanded into their constituent IP ranges. @@ -421,13 +421,13 @@ async def generate_children(self, ssl_verify=False): # Expand seeds first for event_seed in list(self.seeds.event_seeds): - children = await event_seed._generate_children(ssl_verify=ssl_verify) + children = await event_seed._generate_children(helpers=helpers) for child in children: self.seeds.add(child) # Also expand blacklist event seeds (like ASN targets) for event_seed in list(self.blacklist.event_seeds): - children = await event_seed._generate_children(ssl_verify=ssl_verify) + children = await event_seed._generate_children(helpers=helpers) for child in children: self.blacklist.add(child) diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index b757966bb7..b043635b90 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -1196,3 +1196,71 @@ def __init__(self, bbot_io_api_key=None, verify=True): scan2 = bbot_scanner("8.8.8.8") _ = scan2.helpers.asn.client assert captured["bbot_io_api_key"] is None + + +@pytest.mark.asyncio +async def test_asn_helper_circuit_breaker(bbot_scanner, monkeypatch): + """ASNHelper should stop making requests after consecutive failures.""" + from unittest.mock import AsyncMock, MagicMock + + import asndb + + mock_client = MagicMock() + mock_client.lookup_ip = AsyncMock(side_effect=Exception("connection refused")) + monkeypatch.setattr(asndb, "ASNDB", lambda **kw: mock_client) + + scan = bbot_scanner("8.8.8.8") + asn_helper = scan.helpers.asn + assert asn_helper.FAILURE_THRESHOLD == 5 + + # First FAILURE_THRESHOLD calls should each hit the network and return UNKNOWN_ASN + for i in range(asn_helper.FAILURE_THRESHOLD): + result = await asn_helper.ip_to_subnets(f"1.2.3.{i}") + assert result == asn_helper.UNKNOWN_ASN + assert not asn_helper._circuit_broken or i == asn_helper.FAILURE_THRESHOLD - 1 + + # Circuit should now be broken + assert asn_helper._circuit_broken + assert mock_client.lookup_ip.call_count == asn_helper.FAILURE_THRESHOLD + + # Subsequent calls should return immediately without hitting the network + for i in range(10): + result = await asn_helper.ip_to_subnets(f"5.6.7.{i}") + assert result == asn_helper.UNKNOWN_ASN + assert mock_client.lookup_ip.call_count == asn_helper.FAILURE_THRESHOLD + + +@pytest.mark.asyncio +async def test_asn_helper_circuit_breaker_resets_on_success(bbot_scanner, monkeypatch): + """A successful lookup should reset the consecutive failure counter.""" + from unittest.mock import AsyncMock, MagicMock + + import asndb + + call_count = 0 + + async def lookup_ip_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count <= 3: + raise Exception("connection refused") + return {"asn": 15169, "subnets": ["8.8.8.0/24"], "asn_name": "GOOGLE", "org": "Google", "country": "US"} + + mock_client = MagicMock() + mock_client.lookup_ip = AsyncMock(side_effect=lookup_ip_side_effect) + monkeypatch.setattr(asndb, "ASNDB", lambda **kw: mock_client) + + scan = bbot_scanner("8.8.8.8") + asn_helper = scan.helpers.asn + + # 3 failures + for i in range(3): + await asn_helper.ip_to_subnets(f"1.2.3.{i}") + assert asn_helper._consecutive_failures == 3 + assert not asn_helper._circuit_broken + + # 1 success should reset the counter + result = await asn_helper.ip_to_subnets("8.8.8.8") + assert result["asn"] == 15169 + assert asn_helper._consecutive_failures == 0 + assert not asn_helper._circuit_broken diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 30786b5b62..2df98df652 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -383,6 +383,7 @@ async def test_asn_targets(bbot_scanner): assert "ASN:15169" in target.seeds.inputs # Test ASN target expansion with real asndb (Google's AS15169) + scan = bbot_scanner("ASN:15169") target = BBOTTarget(target=["ASN:15169"]) # Verify initial state @@ -390,7 +391,7 @@ async def test_asn_targets(bbot_scanner): initial_seeds = len(target.seeds.event_seeds) # Generate children (expand ASN to IP ranges) - await target.generate_children() + await target.generate_children(helpers=scan.helpers) # After expansion, should have additional IP range seeds assert len(target.seeds.event_seeds) > initial_seeds @@ -474,7 +475,8 @@ async def test_asn_targets_edge_cases(bbot_scanner): initial_seeds = len(target.seeds.event_seeds) with patch("asndb.ASNDB", return_value=mock_empty_client): - await target.generate_children() + scan = bbot_scanner("ASN:99999") + await target.generate_children(helpers=scan.helpers) # Should not add any new seeds for empty ASN assert len(target.seeds.event_seeds) == initial_seeds @@ -582,6 +584,53 @@ async def test_asn_event_json_serialization(bbot_scanner): assert reconstructed.data == {"asn": 12345} +@pytest.mark.asyncio +async def test_asn_resolution_failure_aborts_scan(bbot_scanner): + """When the asndb API is unreachable, the scan should abort gracefully with a helpful message.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from bbot.errors import ScanError + + mock_client = MagicMock() + mock_client.lookup_asn = AsyncMock(side_effect=Exception("connection refused")) + mock_client.cleanup = AsyncMock() + + with patch("asndb.ASNDB", return_value=mock_client): + scan = bbot_scanner("ASN:15169") + with pytest.raises(ScanError, match="Failed to resolve ASN target"): + await scan._prep() + + # Should have retried 3 times + assert mock_client.lookup_asn.call_count == 3 + + +@pytest.mark.asyncio +async def test_asn_resolution_failure_retries(bbot_scanner): + """ASN resolution should succeed if a retry works after initial failures.""" + from unittest.mock import AsyncMock, MagicMock, patch + + call_count = 0 + + async def lookup_asn_side_effect(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise Exception("connection refused") + return {"asn": 15169, "subnets": ["8.8.8.0/24"]} + + mock_client = MagicMock() + mock_client.lookup_asn = AsyncMock(side_effect=lookup_asn_side_effect) + mock_client.cleanup = AsyncMock() + + with patch("asndb.ASNDB", return_value=mock_client): + scan = bbot_scanner("ASN:15169") + await scan._prep() + + # Should have succeeded on the 3rd attempt + assert call_count == 3 + assert "8.8.8.0/24" in scan.preset.target.seeds.hosts + + @pytest.mark.asyncio async def test_blacklist_regex(bbot_scanner, bbot_httpserver): from bbot.scanner.target import ScanBlacklist diff --git a/bbot/test/test_step_2/module_tests/test_module_asn.py b/bbot/test/test_step_2/module_tests/test_module_asn.py index 1f74b776ce..5c848ab9d5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_asn.py +++ b/bbot/test/test_step_2/module_tests/test_module_asn.py @@ -1,3 +1,5 @@ +from unittest.mock import AsyncMock, patch + from .base import ModuleTestBase @@ -28,3 +30,37 @@ def check(self, module_test, events): assert not asn_events, ( f"Should not emit any ASN events for private IP, but found: {[e.data for e in asn_events]}" ) + + +class TestASNReportNoNetwork(ModuleTestBase): + """Regression: report() uses metadata stored during handle_event() with zero network calls.""" + + targets = ["8.8.8.8"] + module_name = "asn" + modules_overrides = ["asn", "speculate"] + config_overrides = {"scope": {"report_distance": 2}, "speculate": True} + + async def check(self, module_test, events): + asn_events = [e for e in events if e.type == "ASN"] + assert asn_events, "No ASN event produced" + + asn_module = module_test.scan.modules["asn"] + + # handle_event() should have stored metadata for report() + assert asn_module.asn_metadata, "asn_metadata is empty after scan" + for asn_number, metadata in asn_module.asn_metadata.items(): + assert isinstance(asn_number, int), f"asn_metadata key {asn_number!r} should be int" + assert metadata["subnet_count"] > 0 + assert metadata["name"] + assert metadata["description"] + + # report() should not touch the network at all + asn_helper = module_test.scan.helpers.asn + with patch.object(asn_helper.client, "request", new_callable=AsyncMock) as mock_request: + await asn_module.report() + mock_request.assert_not_called() + + # Verify the report table was emitted via logging + log_text = "\n".join(r.message for r in module_test.caplog.records) + for asn_number in asn_module.asn_metadata: + assert f"AS{asn_number}" in log_text, f"AS{asn_number} not found in report log output"