Graceful asndb failure handling and report-phase cache fix

bbot/core/event/helpers.py

@@ -111,7 +111,7 @@ def _sanitize_and_extract_host(self, data):
         """
         return data, None, None
 
-    async def _generate_children(self, ssl_verify=False):
+    async def _generate_children(self, helpers):
         return []
 
     def _override_input(self, input):
@@ -290,23 +290,15 @@ class ASN(BaseEventSeed):
     def _override_input(self, input):
         return f"ASN:{self.data}"
 
-    # ASNs are essentially just a superset of IP_RANGES.
-    # This method resolves the ASN to a list of IP_RANGES using the ASN API, and then adds the cidr string as a child event seed.
-    # These will later be automatically resolved to an IP_RANGE event seed and added to the target.
-    async def _generate_children(self, ssl_verify=False):
-        from asndb import ASNDB
-
-        client = ASNDB(verify=ssl_verify)
-        asn_data = await client.lookup_asn(str(self.data), include_subnets=True)
-        children = []
-        if asn_data:
-            subnets = asn_data.get("subnets")
-            if isinstance(subnets, str):
-                subnets = [subnets]
-            if subnets:
-                for cidr in subnets:
-                    children.append(cidr)
-        return children
+    # ASNs are essentially a superset of IP_RANGES. This resolves the ASN to its
+    # subnets via the shared ASN helper and emits each CIDR as a child seed,
+    # which is later resolved to an IP_RANGE seed and added to the target.
+    async def _generate_children(self, helpers):
+        asn_data = await helpers.asn.asn_to_subnets(self.data)
+        subnets = asn_data.get("subnets") or []
+        if isinstance(subnets, str):
+            subnets = [subnets]
+        return list(subnets)
 
     @staticmethod
     def handle_match(match):

bbot/core/helpers/asn.py

@@ -1,5 +1,7 @@
 import logging
 
+from bbot.errors import ASNResolutionError
+
 log = logging.getLogger("bbot.core.helpers.asn")
 
 
@@ -19,9 +21,15 @@ class ASNHelper:
         "country": "Unknown",
     }
 
+    FAILURE_THRESHOLD = 5
+    MAX_RETRIES = 3
+    RETRY_DELAY = 3
+
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self._client = None
+        self._consecutive_failures = 0
+        self._circuit_broken = False
 
     @property
     def client(self):
@@ -45,29 +53,50 @@ def _normalize(self, response):
             "country": response.get("country") or "",
         }
 
+    def _record_failure(self):
+        self._consecutive_failures += 1
+        if self._consecutive_failures >= self.FAILURE_THRESHOLD and not self._circuit_broken:
+            self._circuit_broken = True
+            log.critical(
+                "ASN lookups disabled for the rest of this scan after %d consecutive failures. "
+                "The bbot.io ASN API could not be reached (this may be due to regional network restrictions). "
+                "ASN enrichment data will not be available.",
+                self.FAILURE_THRESHOLD,
+            )
+
     async def ip_to_subnets(self, ip):
         """Return ASN info for an IP address."""
+        if self._circuit_broken:
+            return self.UNKNOWN_ASN
         try:
             response = await self.client.lookup_ip(str(ip), include_subnets=True)
         except Exception as e:
             log.warning(f"ASN lookup failed for IP {ip}: {e}")
+            self._record_failure()
             return self.UNKNOWN_ASN
+        self._consecutive_failures = 0
         return self._normalize(response)
 
     async def asn_to_subnets(self, asn):
-        """Return ASN info (including subnets) for an ASN number."""
-        if isinstance(asn, str):
+        """Resolve an ASN number to its subnets, retrying on transient failure.
+
+        Used by ASN-as-target seed expansion, which cannot degrade gracefully:
+        without the ASN's subnets there is nothing to scan. Raises
+        ASNResolutionError if the ASN can't be resolved after MAX_RETRIES so the
+        scan aborts with guidance instead of silently scanning nothing.
+        """
+        asn = int(str(asn).lower().lstrip("as"))
+        last_error = None
+        for attempt in range(1, self.MAX_RETRIES + 1):
             try:
-                asn = int(asn.lower().lstrip("as"))
-            except ValueError:
-                log.warning(f"Invalid ASN format: {asn}")
-                return self.UNKNOWN_ASN
-        try:
-            response = await self.client.lookup_asn(str(asn), include_subnets=True)
-        except Exception as e:
-            log.warning(f"ASN lookup failed for AS{asn}: {e}")
-            return self.UNKNOWN_ASN
-        return self._normalize(response)
+                response = await self.client.lookup_asn(asn, include_subnets=True)
+                return self._normalize(response)
+            except Exception as e:
+                last_error = e
+                log.warning(f"ASN resolution attempt {attempt}/{self.MAX_RETRIES} failed for AS{asn}: {e}")
+                if attempt < self.MAX_RETRIES:
+                    await self.parent_helper.sleep(self.RETRY_DELAY)
+        raise ASNResolutionError(f"AS{asn}: {last_error}")
 
     async def cleanup(self):
         """Clean up the asndb client."""

bbot/errors.py

@@ -76,3 +76,7 @@ class DNSError(BBOTEngineError):
 
 class ExcavateError(BBOTError):
     pass
+
+
+class ASNResolutionError(BBOTError):
+    pass

bbot/modules/report/asn.py

@@ -18,8 +18,7 @@ class asn(BaseReportModule):
 
     async def setup(self):
         self.unknown_asn = ASNHelper.UNKNOWN_ASN
-        # Track ASN counts locally for reporting
-        self.asn_counts = {}  # ASN number -> count mapping
+        self.asn_metadata = {}
         return True
 
     async def filter_event(self, event):
@@ -41,10 +40,14 @@ async def handle_event(self, event):
             asn_country = asn_data.get("country", "")
             subnets = asn_data.get("subnets", [])
 
-            # Track ASN subnet counts for reporting (only once per ASN)
             if asn_number and asn_number != 0:
-                if asn_number not in self.asn_counts:
-                    self.asn_counts[asn_number] = len(subnets)
+                if asn_number not in self.asn_metadata:
+                    self.asn_metadata[asn_number] = {
+                        "subnet_count": len(subnets),
+                        "name": asn_name,
+                        "description": asn_description,
+                        "country": asn_country,
+                    }
 
             # Don't emit ASN 0 - it's reserved and indicates unknown ASN data
             if asn_number != 0:
@@ -56,34 +59,21 @@ async def handle_event(self, event):
                     )
 
     async def report(self):
-        """Generate an ASN summary table based on locally tracked ASN counts."""
-
-        if not self.asn_counts:
+        if not self.asn_metadata:
             return
 
-        # Build table rows sorted by ASN number (low to high)
-        sorted_asns = sorted(self.asn_counts.items(), key=lambda x: int(x[0]))
+        sorted_asns = sorted(self.asn_metadata.items(), key=lambda x: int(x[0]))
 
         header = ["ASN", "Subnet Count", "Name", "Description", "Country"]
         table = []
-        for asn_number, subnet_count in sorted_asns:
-            # Get ASN details from helper
-            asn_data = await self.helpers.asn.asn_to_subnets(asn_number)
-            if asn_data:
-                asn_name = asn_data.get("name", "")
-                asn_description = asn_data.get("description", "")
-                asn_country = asn_data.get("country", "")
-            else:
-                asn_name = asn_description = asn_country = "unknown"
-
-            number = f"AS{asn_number}" if asn_number != 0 else str(asn_number)
+        for asn_number, metadata in sorted_asns:
             table.append(
                 [
-                    number,
-                    f"{subnet_count:,}",
-                    asn_name,
-                    asn_description,
-                    asn_country,
+                    f"AS{asn_number}",
+                    f"{metadata['subnet_count']:,}",
+                    metadata["name"],
+                    metadata["description"],
+                    metadata["country"],
                 ]
             )
 

bbot/scanner/scanner.py

@@ -20,7 +20,7 @@
 from bbot.core.multiprocess import SHARED_INTERPRETER_STATE
 from bbot.core.helpers.async_helpers import async_to_sync_gen
 from bbot.logger import log_to_stderr
-from bbot.errors import BBOTError, ScanError, ValidationError
+from bbot.errors import ASNResolutionError, BBOTError, ScanError, ValidationError
 from bbot.constants import (
     get_scan_status_code,
     get_scan_status_name,
@@ -327,9 +327,15 @@ async def _prep(self):
         Expands async seed types (e.g. ASN → IP ranges), evaluates preset conditions,
         creates the scan's output folder, loads its modules, and calls their .setup() methods.
         """
-        # expand async seed types (e.g. ASN → IP ranges)
-        ssl_verify = self.preset.web_config.get("ssl_verify_infrastructure", True)
-        await self.preset.target.generate_children(ssl_verify=ssl_verify)
+        # expand async seed types (e.g. ASN -> IP ranges)
+        try:
+            await self.preset.target.generate_children(helpers=self.helpers)
+        except ASNResolutionError as e:
+            raise ScanError(
+                f"Failed to resolve ASN target ({e}). "
+                f"The bbot.io ASN API could not be reached; this may be due to regional network restrictions or a temporary outage. "
+                f"To scan this ASN's networks, look up its prefixes (e.g. at bgp.tools) and pass them directly: bbot -t 1.2.3.0/24 5.6.0.0/16"
+            )
 
         # evaluate preset conditions (may abort the scan)
         if self.preset.conditions:

bbot/scanner/target.py

@@ -410,7 +410,7 @@ def in_target(self, host):
     def __eq__(self, other):
         return self.hash == other.hash
 
-    async def generate_children(self, ssl_verify=False):
+    async def generate_children(self, helpers):
         """
         Generate children for the target, for seed types that expand into other seed types.
         E.g. ASN targets are expanded into their constituent IP ranges.
@@ -421,13 +421,13 @@ async def generate_children(self, ssl_verify=False):
 
         # Expand seeds first
         for event_seed in list(self.seeds.event_seeds):
-            children = await event_seed._generate_children(ssl_verify=ssl_verify)
+            children = await event_seed._generate_children(helpers=helpers)
             for child in children:
                 self.seeds.add(child)
 
         # Also expand blacklist event seeds (like ASN targets)
         for event_seed in list(self.blacklist.event_seeds):
-            children = await event_seed._generate_children(ssl_verify=ssl_verify)
+            children = await event_seed._generate_children(helpers=helpers)
             for child in children:
                 self.blacklist.add(child)
 

bbot/test/test_step_1/test_helpers.py

@@ -1196,3 +1196,71 @@ def __init__(self, bbot_io_api_key=None, verify=True):
     scan2 = bbot_scanner("8.8.8.8")
     _ = scan2.helpers.asn.client
     assert captured["bbot_io_api_key"] is None
+
+
+@pytest.mark.asyncio
+async def test_asn_helper_circuit_breaker(bbot_scanner, monkeypatch):
+    """ASNHelper should stop making requests after consecutive failures."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import asndb
+
+    mock_client = MagicMock()
+    mock_client.lookup_ip = AsyncMock(side_effect=Exception("connection refused"))
+    monkeypatch.setattr(asndb, "ASNDB", lambda **kw: mock_client)
+
+    scan = bbot_scanner("8.8.8.8")
+    asn_helper = scan.helpers.asn
+    assert asn_helper.FAILURE_THRESHOLD == 5
+
+    # First FAILURE_THRESHOLD calls should each hit the network and return UNKNOWN_ASN
+    for i in range(asn_helper.FAILURE_THRESHOLD):
+        result = await asn_helper.ip_to_subnets(f"1.2.3.{i}")
+        assert result == asn_helper.UNKNOWN_ASN
+        assert not asn_helper._circuit_broken or i == asn_helper.FAILURE_THRESHOLD - 1
+
+    # Circuit should now be broken
+    assert asn_helper._circuit_broken
+    assert mock_client.lookup_ip.call_count == asn_helper.FAILURE_THRESHOLD
+
+    # Subsequent calls should return immediately without hitting the network
+    for i in range(10):
+        result = await asn_helper.ip_to_subnets(f"5.6.7.{i}")
+        assert result == asn_helper.UNKNOWN_ASN
+    assert mock_client.lookup_ip.call_count == asn_helper.FAILURE_THRESHOLD
+
+
+@pytest.mark.asyncio
+async def test_asn_helper_circuit_breaker_resets_on_success(bbot_scanner, monkeypatch):
+    """A successful lookup should reset the consecutive failure counter."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    import asndb
+
+    call_count = 0
+
+    async def lookup_ip_side_effect(*args, **kwargs):
+        nonlocal call_count
+        call_count += 1
+        if call_count <= 3:
+            raise Exception("connection refused")
+        return {"asn": 15169, "subnets": ["8.8.8.0/24"], "asn_name": "GOOGLE", "org": "Google", "country": "US"}
+
+    mock_client = MagicMock()
+    mock_client.lookup_ip = AsyncMock(side_effect=lookup_ip_side_effect)
+    monkeypatch.setattr(asndb, "ASNDB", lambda **kw: mock_client)
+
+    scan = bbot_scanner("8.8.8.8")
+    asn_helper = scan.helpers.asn
+
+    # 3 failures
+    for i in range(3):
+        await asn_helper.ip_to_subnets(f"1.2.3.{i}")
+    assert asn_helper._consecutive_failures == 3
+    assert not asn_helper._circuit_broken
+
+    # 1 success should reset the counter
+    result = await asn_helper.ip_to_subnets("8.8.8.8")
+    assert result["asn"] == 15169
+    assert asn_helper._consecutive_failures == 0
+    assert not asn_helper._circuit_broken