diff --git a/bbot/core/event/__init__.py b/bbot/core/event/__init__.py index ea44bf21c9..89b54b5084 100644 --- a/bbot/core/event/__init__.py +++ b/bbot/core/event/__init__.py @@ -1,3 +1,3 @@ -from .base import make_event, update_event, is_event, event_from_json +from .base import make_event, is_event, event_from_json -__all__ = ["make_event", "update_event", "is_event", "event_from_json"] +__all__ = ["make_event", "is_event", "event_from_json"] diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 83331b94d2..a6578d98ba 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -789,32 +789,26 @@ def data_json(self): def __contains__(self, other): """ - Membership checks for Events. - - Supports: - - some_event in other_event (event vs event) - - "host:port" in other_event (string coerced to an event) + Allows events to be compared using the "in" operator: + E.g.: + if some_event in other_event: + ... """ - # Fast path: already an Event - if is_event(other): - other_event = other - else: - try: - other_event = make_event(other, dummy=True) - except ValidationError: - return False - + try: + other = make_event(other, dummy=True) + except ValidationError: + return False # if hashes match - if other_event == self: + if other == self: return True - # if hosts match (including subnet / domain containment) - if self.host and other_event.host: - if self.host == other_event.host: + # if hosts match + if self.host and other.host: + if self.host == other.host: return True # hostnames and IPs radixtarget = RadixTarget() radixtarget.insert(self.host) - return bool(radixtarget.search(other_event.host)) + return bool(radixtarget.search(other.host)) return False def json(self, mode="json", siem_friendly=False): @@ -1002,14 +996,10 @@ def __gt__(self, other): return self.priority > getattr(other, "priority", (0,)) def __eq__(self, other): - """ - Event equality is **only** defined between Event instances. - - Equality is based on the event hash (derived from its id). Comparisons to - non-Event types raise a ValueError to make incorrect comparisons explicit. - """ - if not is_event(other): - raise ValueError("Event equality is only defined between Event instances") + try: + other = make_event(other, dummy=True) + except ValidationError: + return False return hash(self) == hash(other) def __hash__(self): @@ -1758,55 +1748,6 @@ def _pretty_string(self): return self.data["url"] -def update_event( - event, - parent=None, - context=None, - module=None, - scan=None, - tags=None, - internal=None, -): - """ - Updates an existing event object with additional metadata. - - Parameters: - event (BaseEvent): The event object to update. - parent (BaseEvent, optional): New parent event. - context (str, optional): Discovery context to set. - module (str or BaseModule, optional): Module that discovered the event. - scan (Scan, optional): BBOT Scan object associated with the event. - tags (Union[str, List[str]], optional): Tags to merge into the event. - internal (Any, optional): Marks the event as internal if True. - - Returns: - BaseEvent: The updated event object. - """ - if not is_event(event): - raise ValidationError(f"update_event() expects an Event, got {type(event)}") - - # allow tags to be either a string or an array - if not tags: - tags = [] - elif isinstance(tags, str): - tags = [tags] - tags = set(tags) - - if scan is not None and not event.scan: - event.scan = scan - if module is not None: - event.module = module - if parent is not None: - event.parent = parent - if context is not None: - event.discovery_context = context - if internal is True: - event.internal = True - if tags: - event.tags = tags.union(event.tags) - return event - - def make_event( data, event_type=None, @@ -1820,13 +1761,14 @@ def make_event( internal=None, ): """ - Creates and returns a new event object. + Creates and returns a new event object or modifies an existing one. - This function serves as a factory for creating new event objects from raw data. - If you need to modify an existing event, use ``update_event()`` instead. + This function serves as a factory for creating new event objects, either by generating a new `Event` + object or by updating an existing event with additional metadata. If `data` is already an event, + it updates the event based on the additional parameters provided. Parameters: - data (Union[str, dict]): The primary data for the event. + data (Union[str, dict, BaseEvent]): The primary data for the event or an existing event object. event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided. parent (BaseEvent, optional): Parent event leading to this event's discovery. context (str, optional): Description of circumstances leading to event's discovery. @@ -1839,20 +1781,32 @@ def make_event( internal (Any, optional): Makes the event internal if set to True. Defaults to None. Returns: - BaseEvent: A new event object. + BaseEvent: A new or updated event object. Raises: ValidationError: Raised when there's an error in event data or type sanitization. + + Examples: + If inside a module, e.g. from within its `handle_event()`: + >>> self.make_event("1.2.3.4", parent=event) + IP_ADDRESS("1.2.3.4", module=portscan, tags={'ipv4', 'distance-1'}) + + If you're outside a module but you have a scan object: + >>> scan.make_event("1.2.3.4", parent=scan.root_event) + IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4', 'distance-1'}) + + If you're outside a scan and just messing around: + >>> from bbot.core.event.base import make_event + >>> make_event("1.2.3.4", dummy=True) + IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4'}) + + Note: + When working within a module's `handle_event()`, use the instance method + `self.make_event()` instead of calling this function directly. """ if not data: raise ValidationError("No data provided") - # do not allow passing an existing event here – use update_event() instead - if is_event(data): - raise ValidationError( - "make_event() does not accept an existing event object. Use update_event(event, ...) to modify an event." - ) - # allow tags to be either a string or an array if not tags: tags = [] @@ -1860,58 +1814,76 @@ def make_event( tags = [tags] tags = set(tags) - # if event_type is not provided, autodetect it - if event_type is None: - event_seed = EventSeed(data) - event_type = event_seed.type - data = event_seed.data - if not dummy: - log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"') - - event_type = str(event_type).strip().upper() - - # Catch these common whoopsies - if event_type in ("DNS_NAME", "IP_ADDRESS"): - # DNS_NAME <--> EMAIL_ADDRESS confusion - if validators.soft_validate(data, "email"): + # if data is already an event, update it with the user's kwargs + if is_event(data): + event = data # this line used to be event = copy(data), and I don't know why. but it was breaking shit. because make_event() was being called multiple times and making shadow copies of events that were slightly different, i.e. one being internal and the other not. + if scan is not None and not event.scan: + event.scan = scan + if module is not None: + event.module = module + if parent is not None: + event.parent = parent + if context is not None: + event.discovery_context = context + if internal is True: + event.internal = True + if tags: + event.tags = tags.union(event.tags) + event_type = data.type + return event + else: + # if event_type is not provided, autodetect it + if event_type is None: + event_seed = EventSeed(data) + event_type = event_seed.type + data = event_seed.data + if not dummy: + log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"') + + event_type = str(event_type).strip().upper() + + # Catch these common whoopsies + if event_type in ("DNS_NAME", "IP_ADDRESS"): + # DNS_NAME <--> EMAIL_ADDRESS confusion + if validators.soft_validate(data, "email"): + event_type = "EMAIL_ADDRESS" + else: + # DNS_NAME <--> IP_ADDRESS confusion + try: + data = validators.validate_host(data) + except Exception as e: + log.trace(traceback.format_exc()) + raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}') + data_is_ip = is_ip(data) + if event_type == "DNS_NAME" and data_is_ip: + event_type = "IP_ADDRESS" + elif event_type == "IP_ADDRESS" and not data_is_ip: + event_type = "DNS_NAME" + # USERNAME <--> EMAIL_ADDRESS confusion + if event_type == "USERNAME" and validators.soft_validate(data, "email"): event_type = "EMAIL_ADDRESS" - else: - # DNS_NAME <--> IP_ADDRESS confusion - try: - data = validators.validate_host(data) - except Exception as e: - log.trace(traceback.format_exc()) - raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}') - data_is_ip = is_ip(data) - if event_type == "DNS_NAME" and data_is_ip: - event_type = "IP_ADDRESS" - elif event_type == "IP_ADDRESS" and not data_is_ip: - event_type = "DNS_NAME" - # USERNAME <--> EMAIL_ADDRESS confusion - if event_type == "USERNAME" and validators.soft_validate(data, "email"): - event_type = "EMAIL_ADDRESS" - tags.add("affiliate") - # Convert single-host IP_RANGE to IP_ADDRESS - if event_type == "IP_RANGE": - with suppress(Exception): - net = ipaddress.ip_network(data, strict=False) - if net.prefixlen == net.max_prefixlen: - event_type = "IP_ADDRESS" - data = net.network_address - - event_class = globals().get(event_type, DefaultEvent) - return event_class( - data, - event_type=event_type, - parent=parent, - context=context, - module=module, - scan=scan, - tags=tags, - confidence=confidence, - _dummy=dummy, - _internal=internal, - ) + tags.add("affiliate") + # Convert single-host IP_RANGE to IP_ADDRESS + if event_type == "IP_RANGE": + with suppress(Exception): + net = ipaddress.ip_network(data, strict=False) + if net.prefixlen == net.max_prefixlen: + event_type = "IP_ADDRESS" + data = net.network_address + + event_class = globals().get(event_type, DefaultEvent) + return event_class( + data, + event_type=event_type, + parent=parent, + context=context, + module=module, + scan=scan, + tags=tags, + confidence=confidence, + _dummy=dummy, + _internal=internal, + ) def event_from_json(j, siem_friendly=False): diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 4387a6f9dc..df35df79ff 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -7,7 +7,6 @@ from ..core.helpers.misc import get_size # noqa from ..errors import ValidationError, WebError from ..core.helpers.async_helpers import TaskCounter, ShuffleQueue -from ..core.event import is_event class BaseModule: @@ -520,12 +519,6 @@ def make_event(self, *args, **kwargs): if (not args) or getattr(args[0], "module", None) is None: kwargs["module"] = self try: - if args and is_event(args[0]): - raise ValidationError( - f"{self.__class__.__name__}.make_event() does not accept an existing event " - f"({type(args[0]).__name__}) as the first argument. " - "Use update_event(event, ...) or emit_event(event, ...) instead." - ) event = self.scan.make_event(*args, **kwargs) except ValidationError as e: if raise_error: @@ -534,39 +527,6 @@ def make_event(self, *args, **kwargs): return return event - def update_event(self, event, **kwargs): - """Update an existing event for the scan. - - This is the counterpart to :meth:`make_event` for modifying an existing - :class:`bbot.core.event.base.BaseEvent` instance. - - Raises a validation error if the update could not be applied, unless - ``raise_error`` is set to False. - - Args: - event: The event object to update. - **kwargs: Keyword arguments to be passed to the scan's update_event method. - raise_error (bool, optional): Whether to raise a validation error if the event could not be updated. Defaults to False. - - Returns: - Event or None: The updated event, or None if a validation error occurred and raise_error was False. - - Raises: - ValidationError: If the event could not be validated and raise_error is True. - """ - raise_error = kwargs.pop("raise_error", False) - module = kwargs.pop("module", None) - if module is None and getattr(event, "module", None) is None: - kwargs["module"] = self - try: - updated = self.scan.update_event(event, **kwargs) - except ValidationError as e: - if raise_error: - raise - self.warning(f"{e}") - return - return updated - async def emit_event(self, *args, **kwargs): """Emit an event to the event queue and distribute it to interested modules. @@ -602,23 +562,7 @@ async def emit_event(self, *args, **kwargs): v = event_kwargs.pop(o, None) if v is not None: emit_kwargs[o] = v - - # Two entry points: - # - emit_event(data, ...) -> create a new event via make_event() - # - emit_event(existing_event, ...) -> update and re‑emit that event - if args and is_event(args[0]): - event, *rest = args - if rest: - self.warning( - f"emit_event() was called on {self.name} with an existing event and extra " - f"positional args ({rest}); extra args are ignored. " - "Pass only the event plus keyword arguments, or call make_event() explicitly." - ) - # Update the existing event (e.g. tags/context/module) before emitting - event = self.update_event(event, **event_kwargs) - else: - event = self.make_event(*args, **event_kwargs) - + event = self.make_event(*args, **event_kwargs) if event is not None: children = event.children for e in [event] + children: diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 6d0269cae6..9887f18225 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -33,8 +33,7 @@ async def handle_event(self, event): subdomains = await self.query(query, parse_fn=self.parse_domains, request_fn=self.request_domains) if subdomains: for s in subdomains: - # `s` is a hostname string; compare against the event's data, not the Event object itself. - if s != event.data: + if s != event: await self.emit_event( s, "DNS_NAME", @@ -46,8 +45,7 @@ async def handle_event(self, event): redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects) if redirects: for r in redirects: - # `r` is a hostname string; compare against the event's data, not the Event object itself. - if r != event.data: + if r != event: await self.emit_event( r, "DNS_NAME", diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 3c52cf64fe..be0ab89165 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -77,7 +77,7 @@ async def handle_event(self, event): dns_names = dns_names[:1] + [n for n in dns_names[1:] if self.scan.in_scope(n)] for event_type, results in (("DNS_NAME", set(dns_names)), ("EMAIL_ADDRESS", emails)): for event_data in results: - if event_data is not None and event_data != event.data: + if event_data is not None and event_data != event: self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]") try: ssl_event = self.make_event(event_data, event_type, parent=event, raise_error=True) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 67c09ea38d..a7b59bb117 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -10,7 +10,7 @@ from collections import OrderedDict from bbot import __version__ -from bbot.core.event import make_event, update_event +from bbot.core.event import make_event from .manager import ScanIngress, ScanEgress from bbot.core.helpers.misc import sha1, rand_string from bbot.core.helpers.names_generator import random_name @@ -995,10 +995,6 @@ def make_event(self, *args, **kwargs): event = make_event(*args, **kwargs) return event - def update_event(self, event, **kwargs): - kwargs["scan"] = self - return update_event(event, **kwargs) - @property def root_event(self): """ diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index b4bb8582eb..3cf230b0ae 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -42,7 +42,7 @@ async def test_events(events, helpers): assert events.ipv4 == scan.make_event("8.8.8.8", dummy=True) assert "8.8.8.8" in events.ipv4 assert events.ipv4.host_filterable == "8.8.8.8" - assert events.ipv4.data == "8.8.8.8" + assert "8.8.8.8" == events.ipv4 assert "8.8.8.8" in events.netv4 assert "8.8.8.9" not in events.ipv4 assert "8.8.9.8" not in events.netv4 @@ -60,7 +60,7 @@ async def test_events(events, helpers): assert events.emoji not in events.netv6 assert events.netv6 not in events.emoji ipv6_event = scan.make_event(" [DEaD::c0De]:88", "DNS_NAME", dummy=True) - assert ipv6_event.data == "dead::c0de" + assert "dead::c0de" == ipv6_event assert ipv6_event.host_filterable == "dead::c0de" range_to_ip = scan.make_event("1.2.3.4/32", dummy=True) assert range_to_ip.type == "IP_ADDRESS" @@ -87,7 +87,7 @@ async def test_events(events, helpers): open_port_event = scan.make_event(" eViLcorp.COM.:88", "DNS_NAME", dummy=True) dns_event = scan.make_event("evilcorp.com.", "DNS_NAME", dummy=True) for e in (open_port_event, dns_event): - assert e.data == "evilcorp.com" + assert "evilcorp.com" == e assert e.netloc == "evilcorp.com" assert e.json()["netloc"] == "evilcorp.com" assert e.port is None @@ -117,19 +117,17 @@ async def test_events(events, helpers): assert events.emoji not in events.url_unverified assert events.emoji not in events.ipv6_url_unverified assert events.url_unverified not in events.emoji - - # URL normalization tests – compare against normalized event.data / .with_port().geturl() - assert scan.make_event("https://evilcorp.com:443", dummy=True).data == "https://evilcorp.com/" - assert scan.make_event("http://evilcorp.com:80", dummy=True).data == "http://evilcorp.com/" + assert "https://evilcorp.com" == scan.make_event("https://evilcorp.com:443", dummy=True) + assert "http://evilcorp.com" == scan.make_event("http://evilcorp.com:80", dummy=True) assert "http://evilcorp.com:80/asdf.js" in scan.make_event("http://evilcorp.com/asdf.js", dummy=True) assert "http://evilcorp.com/asdf.js" in scan.make_event("http://evilcorp.com:80/asdf.js", dummy=True) - assert scan.make_event("https://evilcorp.com", dummy=True).data == "https://evilcorp.com/" - assert scan.make_event("http://evilcorp.com", dummy=True).data == "http://evilcorp.com/" - assert scan.make_event("https://evilcorp.com:80", dummy=True).data == "https://evilcorp.com:80/" - assert scan.make_event("http://evilcorp.com:443", dummy=True).data == "http://evilcorp.com:443/" + assert "https://evilcorp.com:443" == scan.make_event("https://evilcorp.com", dummy=True) + assert "http://evilcorp.com:80" == scan.make_event("http://evilcorp.com", dummy=True) + assert "https://evilcorp.com:80" == scan.make_event("https://evilcorp.com:80", dummy=True) + assert "http://evilcorp.com:443" == scan.make_event("http://evilcorp.com:443", dummy=True) assert scan.make_event("https://evilcorp.com", dummy=True).with_port().geturl() == "https://evilcorp.com:443/" assert scan.make_event("https://evilcorp.com:666", dummy=True).with_port().geturl() == "https://evilcorp.com:666/" - assert scan.make_event("https://evilcorp.com.:666", dummy=True).data == "https://evilcorp.com:666/" + assert scan.make_event("https://evilcorp.com.:666", dummy=True) == "https://evilcorp.com:666/" assert scan.make_event("https://[bad::c0de]", dummy=True).with_port().geturl() == "https://[bad::c0de]:443/" assert scan.make_event("https://[bad::c0de]:666", dummy=True).with_port().geturl() == "https://[bad::c0de]:666/" url_event = scan.make_event("https://evilcorp.com", "URL", events.ipv4_url, tags=["status-200"]) @@ -262,20 +260,20 @@ async def test_events(events, helpers): ) assert event.discovery_context == "something discovered IP_ADDRESS: 127.0.0.1" - # updating an already-created event with update_event() + # updating an already-created event with make_event() # updating tags event1 = scan.make_event("127.0.0.1", parent=scan.root_event) - updated_event = scan.update_event(event1, tags="asdf") + updated_event = scan.make_event(event1, tags="asdf") # assert "asdf" not in event1.tags # why was this test added? why is it important the original event stays untouched? 🤔 assert "asdf" in updated_event.tags # updating parent event2 = scan.make_event("127.0.0.1", parent=scan.root_event) - updated_event = scan.update_event(event2, parent=event1) + updated_event = scan.make_event(event2, parent=event1) # assert event2.parent == scan.root_event assert updated_event.parent == event1 - # updating module/internal flag + # updating module event3 = scan.make_event("127.0.0.1", parent=scan.root_event) - updated_event = scan.update_event(event3, internal=True) + updated_event = scan.make_event(event3, internal=True) # assert event3.internal is False assert updated_event.internal is True diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index a915eea57d..1282110400 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -10,7 +10,7 @@ async def test_python_api(): events1 = [] async for event in scan1.async_start(): events1.append(event) - assert any(e.type == "IP_ADDRESS" and e.data == "127.0.0.1" for e in events1) + assert any("127.0.0.1" == e for e in events1) # make sure output files work scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") await scan2.async_start_without_generator() @@ -69,7 +69,7 @@ def test_python_api_sync(): events1 = [] for event in scan1.start(): events1.append(event) - assert any(e.type == "IP_ADDRESS" and e.data == "127.0.0.1" for e in events1) + assert any("127.0.0.1" == e for e in events1) # make sure output files work scan2 = Scanner("127.0.0.1", output_modules=["json"], scan_name="python_api_test") scan2.start_without_generator()