diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2c88fa671..748ededd04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: # if one python version fails, let the others finish fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v6 - name: Set up Python diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 44ddbe7320..61421d4727 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -1271,7 +1271,7 @@ async def handle_event(self, event, **kwargs): content_type = headers["content-type"][0] # skip PDF responses -- running YARA/regex on raw PDF bytes produces false positives and wastes time. - # PDFs are still processed correctly via the filedownload → extractous → RAW_TEXT pipeline, + # PDFs are still processed correctly via the filedownload → kreuzberg → RAW_TEXT pipeline, # which extracts readable text and feeds it back to excavate as a RAW_TEXT event (handled separately below). # TODO: remove this in favor of a proper categorization system for text vs non-text (i.e. to-be-extracted) content if content_type and "application/pdf" in content_type.lower(): diff --git a/bbot/modules/extractous.py b/bbot/modules/kreuzberg.py similarity index 73% rename from bbot/modules/extractous.py rename to bbot/modules/kreuzberg.py index c37be08008..2b04176d11 100644 --- a/bbot/modules/extractous.py +++ b/bbot/modules/kreuzberg.py @@ -1,9 +1,10 @@ -from extractous import Extractor +import pypdfium2 +from kreuzberg import extract_file from bbot.modules.base import BaseModule -class extractous(BaseModule): +class kreuzberg(BaseModule): watched_events = ["FILESYSTEM"] produced_events = ["RAW_TEXT"] flags = ["passive", "safe"] @@ -65,7 +66,7 @@ class extractous(BaseModule): "extensions": "File extensions to parse", } - deps_pip = ["extractous~=0.3.0"] + deps_pip = ["kreuzberg~=4.3", "pypdfium2~=5.0"] scope_distance_modifier = 1 async def setup(self): @@ -82,11 +83,17 @@ async def filter_event(self, event): async def handle_event(self, event): file_path = event.data["path"] - content = await self.scan.helpers.run_in_executor_mp(extract_text, file_path) - if isinstance(content, tuple): - error, traceback = content - self.error(f"Error extracting text from {file_path}: {error}") - self.trace(traceback) + try: + if file_path.lower().endswith(".pdf"): + content = await self.helpers.run_in_executor_mp(self.extract_pdf, file_path) + else: + result = await extract_file(file_path) + content = result.content.strip() + except Exception as e: + import traceback + + self.error(f"Error extracting text from {file_path}: {e}") + self.trace(traceback.format_exc()) return if content: @@ -98,27 +105,22 @@ async def handle_event(self, event): ) await self.emit_event(raw_text_event) + @staticmethod + def extract_pdf(file_path): + """Extract text from PDF using pypdfium2 directly instead of kreuzberg. -def extract_text(file_path): - """ - extract_text Extracts plaintext from a document path using extractous. - - :param file_path: The path of the file to extract text from. - :return: ASCII-encoded plaintext extracted from the document. - """ - - try: - extractor = Extractor() - reader, metadata = extractor.extract_file(str(file_path)) - - result = "" - buffer = reader.read(4096) - while len(buffer) > 0: - result += buffer.decode("utf-8", errors="ignore") - buffer = reader.read(4096) - - return result.strip() - except Exception as e: - import traceback - - return (str(e), traceback.format_exc()) + kreuzberg's bundled pdfium extracts text spatially via get_text_bounded(), which + clips long unbroken strings (JWTs, base64, URLs) that extend beyond the page width. + pypdfium2's get_text_range() extracts by character index, returning complete text + regardless of spatial position. + """ + document = pypdfium2.PdfDocument(file_path) + try: + pages = [] + for page in document: + textpage = page.get_textpage() + text = textpage.get_text_range() + pages.append(text) + return " ".join("\n".join(pages).strip().split()) + finally: + document.close() diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index a7dc794a1e..259146fe41 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -1322,7 +1322,7 @@ def check(self, module_test, events): class TestExcavateRAWTEXT(ModuleTestBase): targets = ["http://127.0.0.1:8888/", "test.notreal"] - modules_overrides = ["excavate", "httpx", "filedownload", "extractous"] + modules_overrides = ["excavate", "httpx", "filedownload", "kreuzberg"] config_overrides = { "scope": {"report_distance": 1}, "web": {"spider_distance": 2, "spider_depth": 2}, @@ -1400,15 +1400,7 @@ class TestExcavateRAWTEXT(ModuleTestBase): startxref 1669 %%EOF""" - extractous_response = """This is an email example@blacklanternsecurity.notreal - -An example JWT eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c - -A serialized DOTNET object AAEAAAD/////AQAAAAAAAAAMAgAAAFJTeXN0ZW0uQ29sbGVjdGlvbnMuR2VuZXJpYy5MaXN0YDFbW1N5c3RlbS5TdHJpbmddXSwgU3lzdGVtLCBWZXJzaW9uPTQuMC4wLjAsIEN1bHR1cmU9bmV1dHJhbCwgUHVibGljS2V5VG9rZW49YjAzZjVmN2YxMWQ1MGFlMwEAAAAIQ29tcGFyZXIQSXRlbUNvdW50AQMAAAAJAwAAAAlTeXN0ZW0uU3RyaW5nW10FAAAACQIAAAAJBAAAAAkFAAAACRcAAAAJCgAAAAkLAAAACQwAAAAJDQAAAAkOAAAACQ8AAAAJEAAAAAkRAAAACRIAAAAJEwAAAA== - -A full url https://www.test.notreal/about - -A href Click me""" + kreuzberg_response = "This is an email example@blacklanternsecurity.notreal An example JWT eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c A serialized DOTNET object AAEAAAD/////AQAAAAAAAAAMAgAAAFJTeXN0ZW0uQ29sbGVjdGlvbnMuR2VuZXJpYy5MaXN0YDFbW1N5c3RlbS5TdHJpbmddXSwgU3lzdGVtLCBWZXJzaW9uPTQuMC4wLjAsIEN1bHR1cmU9bmV1dHJhbCwgUHVibGljS2V5VG9rZW49YjAzZjVmN2YxMWQ1MGFlMwEAAAAIQ29tcGFyZXIQSXRlbUNvdW50AQMAAAAJAwAAAAlTeXN0ZW0uU3RyaW5nW10FAAAACQIAAAAJBAAAAAkFAAAACRcAAAAJCgAAAAkLAAAACQwAAAAJDQAAAAkOAAAACQ8AAAAJEAAAAAkRAAAACRIAAAAJEwAAAA== A full url https://www.test.notreal/about A href Click me" async def setup_after_prep(self, module_test): module_test.set_expect_requests( @@ -1429,13 +1421,13 @@ def check(self, module_test, events): assert open(file).read() == self.pdf_data, f"File at {file} does not contain the correct content" raw_text_events = [e for e in events if e.type == "RAW_TEXT"] assert 1 == len(raw_text_events), "Failed to emit RAW_TEXT event" - assert raw_text_events[0].data == self.extractous_response, ( + assert raw_text_events[0].data == self.kreuzberg_response, ( f"Text extracted from PDF is incorrect, got {raw_text_events[0].data}" ) email_events = [e for e in events if e.type == "EMAIL_ADDRESS"] assert 1 == len(email_events), "Failed to emit EMAIL_ADDRESS event" assert email_events[0].data == "example@blacklanternsecurity.notreal", ( - f"Email extracted from extractous text is incorrect, got {email_events[0].data}" + f"Email extracted from kreuzberg text is incorrect, got {email_events[0].data}" ) finding_events = [e for e in events if e.type == "FINDING"] assert 2 == len(finding_events), "Failed to emit FINDING events" @@ -1460,10 +1452,10 @@ def check(self, module_test, events): assert finding_events[0].data["path"] == str(file), "File path not included in finding event" url_events = [e.data for e in events if e.type == "URL_UNVERIFIED"] assert "https://www.test.notreal/about" in url_events, ( - f"URL extracted from extractous text is incorrect, got {url_events}" + f"URL extracted from kreuzberg text is incorrect, got {url_events}" ) assert "/donot_detect.js" not in url_events, ( - f"URL extracted from extractous text is incorrect, got {url_events}" + f"URL extracted from kreuzberg text is incorrect, got {url_events}" ) diff --git a/bbot/test/test_step_2/module_tests/test_module_extractous.py b/bbot/test/test_step_2/module_tests/test_module_extractous.py deleted file mode 100644 index 19380dbf06..0000000000 --- a/bbot/test/test_step_2/module_tests/test_module_extractous.py +++ /dev/null @@ -1,66 +0,0 @@ -import base64 -from pathlib import Path -from .base import ModuleTestBase - -from ...bbot_fixtures import * - - -class TestExtractous(ModuleTestBase): - targets = ["http://127.0.0.1:8888"] - modules_overrides = ["extractous", "filedownload", "httpx", "excavate", "speculate"] - config_overrides = { - "web": { - "spider_distance": 2, - "spider_depth": 2, - }, - "modules": { - "filedownload": { - "output_folder": bbot_test_dir / "filedownload", - }, - }, - } - - pdf_data = base64.b64decode( - "JVBERi0xLjMKJe+/ve+/ve+/ve+/vSBSZXBvcnRMYWIgR2VuZXJhdGVkIFBERiBkb2N1bWVudCBodHRwOi8vd3d3LnJlcG9ydGxhYi5jb20KMSAwIG9iago8PAovRjEgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL0Jhc2VGb250IC9IZWx2ZXRpY2EgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YxIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKMyAwIG9iago8PAovQ29udGVudHMgNyAwIFIgL01lZGlhQm94IFsgMCAwIDU5NS4yNzU2IDg0MS44ODk4IF0gL1BhcmVudCA2IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iago0IDAgb2JqCjw8Ci9QYWdlTW9kZSAvVXNlTm9uZSAvUGFnZXMgNiAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0F1dGhvciAoYW5vbnltb3VzKSAvQ3JlYXRpb25EYXRlIChEOjIwMjQwNjAzMTg1ODE2KzAwJzAwJykgL0NyZWF0b3IgKFJlcG9ydExhYiBQREYgTGlicmFyeSAtIHd3dy5yZXBvcnRsYWIuY29tKSAvS2V5d29yZHMgKCkgL01vZERhdGUgKEQ6MjAyNDA2MDMxODU4MTYrMDAnMDAnKSAvUHJvZHVjZXIgKFJlcG9ydExhYiBQREYgTGlicmFyeSAtIHd3dy5yZXBvcnRsYWIuY29tKSAKICAvU3ViamVjdCAodW5zcGVjaWZpZWQpIC9UaXRsZSAodW50aXRsZWQpIC9UcmFwcGVkIC9GYWxzZQo+PgplbmRvYmoKNiAwIG9iago8PAovQ291bnQgMSAvS2lkcyBbIDMgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iago3IDAgb2JqCjw8Ci9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvTGVuZ3RoIDEwNwo+PgpzdHJlYW0KR2FwUWgwRT1GLDBVXEgzVFxwTllUXlFLaz90Yz5JUCw7VyNVMV4yM2loUEVNXz9DVzRLSVNpOTBNakdeMixGUyM8UkM1K2MsbilaOyRiSyRiIjVJWzwhXlREI2dpXSY9NVgsWzVAWUBWfj5lbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA4CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA3MyAwMDAwMCBuIAowMDAwMDAwMTA0IDAwMDAwIG4gCjAwMDAwMDAyMTEgMDAwMDAgbiAKMDAwMDAwMDQxNCAwMDAwMCBuIAowMDAwMDAwNDgyIDAwMDAwIG4gCjAwMDAwMDA3NzggMDAwMDAgbiAKMDAwMDAwMDgzNyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzw4MGQ5ZjViOTY0ZmM5OTI4NDUwMWRlYjdhNmE2MzdmNz48ODBkOWY1Yjk2NGZjOTkyODQ1MDFkZWI3YTZhNjM3Zjc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAoaHR0cDovL3d3dy5yZXBvcnRsYWIuY29tKQoKL0luZm8gNSAwIFIKL1Jvb3QgNCAwIFIKL1NpemUgOAo+PgpzdGFydHhyZWYKMTAzNAolJUVPRg==" - ) - - docx_data = base64.b64decode( - "UEsDBBQAAAAIAK+YSUqNEzDqOgEAAKcCAAAQAAAAZG9jUHJvcHMvYXBwLnhtbK2SzU4CMRSFX6Xp3ungghjCDDGwcKHGBMR1be8wjf1Le0Hm2Vz4SL6C7SAM6s7YXc/5eu5P+vH2Pp3tjSY7CFE5W9FRUVICVjip7KaiW2wuriiJyK3k2lmoaAeRzuop95OH4DwEVBBJyrCxoi2inzAWRQuGxyLZNjmNC4ZjuoYNc02jBCyc2BqwyC7Lcsxgj2AlyAt/CqSHxMkO/xoqncj9xfWq80Me9//ZZL+Fa++1EhzT+uo7JYKLrkHy5IIkKZNgC+QVnqfsB5qfpgpLENugsKvLnjhXMrEUXMM8VawbriP0zKBlYu6M57Yj7MC3PIBMKef8ScvETVpH0Mq+xHnL7QbkGfnb+xpwffge9WhclOkchznKmVqB8Zoj1Pd5kbqQDk3PnYxM3ebwR79yi6wMlb/rvTT8rvoTUEsDBBQAAAAIAAVuZllQ34JjWAEAAIwCAAARAAAAZG9jUHJvcHMvY29yZS54bWyNUt1OgzAUfhXSeyiFDWcDLFHjhXGJiUs03tVytuGAkvZMtmfzwkfyFey6wZzxQq4O5/tpvw++Pj7T6bauvHfQplRNRlgQEg8aqYqyWWZkgwt/QqZ5KpWGB61a0FiC8aymMbyQGVkhtpzSdqOrQOklLSSFCmpo0FAWMEoGLoKuzZ8ChwzMrSkHVtd1QRc7XhSGjD7P7h/lCmrhl41B0Ug4qgaFcbAJ7FUbiyyUrgUa59AKuRZL2DsltAYUhUBB98n8dohG8rSQHEusIE/t5frRTmbz+gYSD+vhZQ27TunC2PVptIQCjNRli7bWg+JscayDSw0CofBsaI67FjLSI0/x9c38luRRGI18xvwwmUeMjyac2VrjywsWxi973zOfk3Ftv+Ci/LdzxFnCx+Pg0j7jMPnh3Bu5UO4YpfM7BZU3U7Y6F61fp5UwODsKrnZntF9Q6oo//VL5N1BLAwQUAAAACAAFbmZZnlZ1fjgCAAACCQAAEgAAAHdvcmQvZm9udFRhYmxlLnhtbOWV0W7aMBRAf8Xye4mTAKWoaUWhSJOmPUz7AeM4xFpsR76GwLftYZ+0X9hNSAAVoTaVxsuClIR7fY/t44v48+v34/NOF2QrHShrEhoOGCXSCJsqs07oxmd3E0rAc5PywhqZ0L0E+vz0WE0zazwQrDYw1SKhufflNAhA5FJzGNhSGkxm1mnu8atbB5q7n5vyTlhdcq9WqlB+H0SMjWmLcR+h2CxTQi6s2GhpfFMfOFkg0RrIVQkdrfoIrbIuLZ0VEgB3rIsDT3NljphweAHSSjgLNvMD3Ey7ogaF5SFr3nRxAoz6AaK3gPJzSzjta+F4hY/TisYg+xFH7ZoC2Gu5OwMJlfYjjTsSVp5x+kEmF47HQu4+xwiw8txM6tO8FynqTjyoa7nnOYecEi2mX9bGOr4qUDa2EcFOIPVhkuYA6js6qB/Nq9yRbnra/cBINTVcY/mcF2rlVJMoubEgQ8xteZFQnH/JRnivP0MW13dKgnqkyLkD6Y8jWRvPuFbFvgtDpQDaTKm8yLvEljtVr77NgVpjZgMrltBXhle0XNJDJEzoEAOz+TES1dM1V9hG4mMEl4FraziHEQ/LNhKej8FJg4OGCx0/lJZAvsmKfLeamytaIjZGHSOUUuuJe2pxDbm/lmh2rmWOkfvJML7Q8vC+lmVfLW2XkK9qnfurvRLfuFdmTa+8vumViN2/XEhh/6BXZqW3QBYKyoLvr0h5Qciw1RLdRAr+z6CDyf1JSruX+HZS/lsZ7Qs8/QVQSwMEFAAAAAgABW5mWcFNiUYeBAAADgwAABEAAAB3b3JkL3NldHRpbmdzLnhtbLVWXW7bOBC+iqDndWxJtpMITQv/xNsW8XYRZw9AiSObCH8EkrLjFnuyfdgj7RV2KImRnRhBkqIvNjnfzDcz5HBG//3z74dPD4IHW9CGKXkVRmeDMACZK8rk+iqsbNG7CANjiaSEKwlX4R5M+Onjh11qwFpUMgESSJOK/CrcWFum/b7JNyCIOVMlSAQLpQWxuNXrviD6vip7uRIlsSxjnNl9Px4MxmFLo9CplmlL0RMs18qowjqTVBUFy6H98xb6NX4bk7nKKwHS1h77GjjGoKTZsNJ4NvFeNgQ3nmT7UhJbwb3eLhq8It2d0vTR4jXhOYNSqxyMwQsS3AfIZOd4+Izo0fcZ+m5TrKnQPBrUq8PIR28jiJ8QGP6aTBrohmWa6P2JNMr35dEdzlyTHf51aY0NvI1x1CbWN3sBDwdEOaNvYxp7JrQ84HkbycWzixrn8PA+jj5aHp4MtXTzJqbYl03f2RJLNsTgIxF5+mUtlSYZx8PGWgywnAJ3mUF9Ae4Xz8D91Ut4CLz70HWe70qJYJeWoHN8fti0Bti0+g6xmuT3t7BlrpsZ1NkSrLOCcAOtBoWCVNzekWxlVek1zmPPkG8IcljQq5LkWBkzJa1W3CtS9YeyM+xeGuvHm9TNrFutmsaIJpIITPCo2S0VxVB2aaXZ648y9O6j0ZHPp54U9nHNKGB2HFZ2z2GB4a/Yd5hI+rUyliFl3fN+IoQXIwDpXH/Dl3y3L2EBxFZ4Ur/KW30bC87KJdNa6S+SYj38Om+sKECjB0YsLLGImFa7+qg/A6E4QX/Wcf+wlnAgU+MXt0pZr5sk0/PRYHzRxupgDw0W15NJMr8+Ab1gNVwk04tZkpyAxuPr6XQ6n52Azi/iZLaYDH3kbbwidYPwT+1XrgAD0ZjMiMg0I8GyHpVoJtJM30+Z9AoZYPOHI2hVZR7t9VrECML5Ap+pR5rHK1LKTDmHotnwJdHrjtvr6NNi7AtfH/lcWwH9u1ZV2cI7TcqmvLxONBx6WybtDRMeMFW2erSTOLcOsErSb1vdHFl3Uti3sFDqt3pD6oKrlUH2/lr5iuR65aoJlqQsm6LM1tFVyNl6Y7F6kAJ3FL+u6k22jlssrrG4weoNyV16qN0uOhlqtYtOlnhZ0slw5raLToafA+2ik+HoahdOtsGGoDmT9/g+/NLJC8W52gH93OHPRO0pmA0pYd50cKw11Qjalm6CbQoPOA6AMovfrCWjguDkiwbxuLZv1TnZq8oeKTvMaZfHFG5iPb7NI+u64p9E42ZLzrA0V3uRdRPjrI2dM4MdpcTpYpX24G8NGA1TqvIvbuINTz3XaFTPJXvnxhve/i0UU2KAetAbjxrjH5NoMru8XiS9y+Q87g3Pk6h3OR7Pe1izl4vF5SCeRbO//cP13/Ef/wdQSwMEFAAAAAgABW5mWX+NfPRJDgAA56IAAA8AAAB3b3JkL3N0eWxlcy54bWztXdt227gV/RUuPbUPHlm8Scoaz6zESZpMc/HEns4zREIWY4pUeYnt/lof+kn9hQIgSFEiIZPEtuzMdGWtWLxgAzz77IMrif/++z8//ny3Do1vNEmDODobTX44HRk08mI/iK7PRnm2PJmNjDQjkU/COKJno3uajn7+6cfbF2l2H9LUYMmj9MXtZmKfjVZZtnkxHqfeiq5J+sM68JI4jZfZD168HsfLZeDR8W2c+GPzdHIqfm2S2KNpyjJ7nZBb9mckAddeAy7e0IhdXMbJmmTsMLker0lyk29OGPyGZMEiCIPsnoGfuiVM0gWlKNnr2MvXNMpE+nFCQ4YYR+kq2KQl2m0XtN3HWocF3poEUQUzzFbrcAvg9AMwGwBuSvtBOBJinN6v6V0NyAv8fkhuicRS1nD6gcyaT+TRu2EYY5aybhk/81e9kMySoDFPSzKyIulqZKy9F++vozghi5AZm7FuMOIMLhVDEMD/Zzbgf8RPemeU2Y+4wvzYe02XJA+zlB8mF4k8lEfiz9s4ylLj9gVJvSC4YmVlWa0Dluu7l1EajNiVFf/ReoWSNHuZBqR+8Y08x697aVa78irwWaqx0P6/2NVvJDwbmXZ16jxtnAxJdF2epNHJb5f1XM9GX8nJLxf81IJBn41IcnL5UqQcy+cb7z/1Zv9IZL0hHpMaN8Iyo0zyE5dFMZZ7wAOWOZ2XB19yTgTJs7jMRSCMd3HHDcuzUMACw2UR8NhVuvwQezfUv8zYhbORyIyd/O39RRLECYtBZ6P5XJ68pOvgXeD7NKrdGK0Cn/6+otFvKfW35399K+KIPOHFecR+W9OJ8IYw9d/ceXTDoxK7GhFOzCeegIng9kUebDMXyf9Zgk1KMtoAVpTwUG9M9jHm/THMVoy0ZoAil72nn/TPyTpaTsyTj5STc7ScWO14pJymR8uJtVKOlNP80XMKIp9VBZOusA8BCVkigITqEEBCVAggoRkEkJAEAkh4PAJIODQCaK4PlMVes4KwQMCNWgMF3KgkUMCNOgEF3KgCUMCNiI8CbgR4FHAjnqOA548BXDTDjPdMcFGmD7eM4yyKM2pk9A4ARyIGJnqzIEBeFdIE85wInCLQyQpaH84j4rjhKA66os94z9CIl8YyuM4TNrCiXXQafaMhG5QwiO8zQCRiQrM8iYDOndAlTdhYE4V6OBCVdxmNKF8vED66Idc4MBr5aBOWkJgIUXk262yvuH4ChHevCRuDQVQDBBcsPgQpwF4cxXiVhyFFgX0CuZoAA3QhBA6gByFwAB0IgeNAmYOZScKhrCXhUEaTcA7UUWG2k3Ao20k4lO0kHMB2V0EW0v0myqTHyN95GPMJCv2SXAbXEWFtA0AlJAddjQuSkOuEbFYGH95uPKV+Rq9i/964glR1FRSs+S885Zw9eBDlAKPuwMF0VgGilFYBorRWAQLU9pG1pXkD7h2o53OZL7JWAffoPVySMC8avQDhsYkMpBTeBkmKE0Q7LsKVP/EmLycVEgm35QQUbQtm4YMUtoASE1HOkE2sgQLzu/sNTVgf7kYf6m0chvEt9YGQl1kSFz5X179pdtf/m/WGzTMHaQOjRyOgXPRgfCQb/We6CNkqBxB7b07YkonQADYu3l19/GBcxRveLeXGASG+irMsXuNA5VjiX36ni7+CiviSdZuje9QDv0QNLQm08wBR8xRQsY+CYg3RIAowdasA/Du9X8Qk8UFwF2zkR+g7oyjIS7LehDCZsUB5y8IRoq0kAP9BkoCPKcH0dYVBq408pvniK/UAoe9TbGBGlT7nmRjDFM1hQK9pBw/QgtjBA7QeBKesyuCOjHjeHTzA8+7gwZ73PCRsqaGcoUUCwp64BIQ/sg0DjMM4WeYh0IglIs6KJSLOjHGYr6MU+tACEPnMAhD+yEjPEYAOCvBvCVsSCmNEoMHoEGgwLgQajAiBhmUBsCqohgZYGlRDm6HQUI2DGhrM37ANA9TUUQ0N5m8CDeZvAg3mbwIN5m/Wa4Mul6yhDKx3apgw36thAmufKKPrTZyQ5B6F+Sak1wQxylrAXSTxkr+6EkfFunJIi5eNdkNb5AUejGo21IIrHAeDlgwxrErY+GWMGprb1kJta+keSideNoGMNXp0FYdsOkb1WAd72JfFSyP7TzDpPnb6IbheZcblqpo8qOPwN1AeSlp28nfSdciyzfKueXj6yg/ydVnW5lpe1+qRurFg17U7pN42M3aSOl2TNnN1OyTdNqZ3kk67Jm3mOuuatLH82D0ojtckuWn1iOlBT6o6hQo/nE46pW7N2OyUtM0bp1Zn4bDBaY9PQEyGKkgN0FFKaoBemlLD9BKXGqa7ytQYB+X2hX4LeMXfK5SKHKv1Go0Kwe4eT3/N2WTsPoDZ4z2096xxFaXUaAWyesyK7cQdtTG7ByA1RvdIpMboHpLUGN1ikzJ9vyClhukerdQY3cOWGqN//DJ145epG79MTPwyMfFLp5WgxujeXFBj9JetCZCtTktCjdFPtiZGtiZAtiZAtiZAtpaubC1d2VoY2VoY2VoA2VoA2VoA2VoA2VoA2Q7tCSjTD5OtBZCtBZCtBZCtrStbW1e2Nka2Nka2NkC2NkC2NkC2NkC2NkC2tqZsbYxsbYBsbYBsbYBsHV3ZOrqydTCydTCydQCydQCydQCydQCydQCydTRl62Bk6wBk6wBk6wBk6+rK1tWVrYuRrYuRrQuQrQuQrQuQrQuQrQuQraspWxcjWxcgWxcg2ybGQU+VM6KqVwImA0ZRla8X9Jgik8X6Un9NfWdQdtK/XGqwHu9OvIrjG6P1FUrL6oESLMIgFgPf9w0cxPKLz+f1l5OGfbWk68PIlzfEHG1jQNTunLQxKGObXZM2Ooa21TVpo3Fq212TNipI+2AgFiIt18WwaqqR+rRj6okivdsxfdPQ044pm3aedUzZNPO8Y0rH4BF7P7nT1Vhutfq1ATHpCDFVQ5j9KFNOG3TnTg3RmUQ1RGc21RD9aFXiDOBXjdWfaDXWQMZNfcY1ZKuG6M24CWLcBDJuAhk3UYxb+oxb+oxrRGw1xDDGLSDjFpBxC8W4rc+4rc+4rc+4bmWtxNFg3AYybqMYd/QZd/QZd/QZd0CMO0DGHSDjDopxV59xV59xV59xF8S4C2TcBTLu9mNcjMJodK9q6Xu202ope1bWtZQ9I3Yt5ZDuVS350O5VDWJo96pJ2cDuVZ27gd2rOokDu1d1Ngd2rxq0DuxetfI7sHvVSvTA7pWacVOfcQ3ZDuxetTFughg3gYybQMZNFOOWPuOWPuMaEXtg90rJuAVk3AIybqEYt/UZt/UZt/UZ162sB3avDjJuAxm3UYw7+ow7+ow7+ow7IMYdIOMOkHEHxbirz7irz7irz7gLYtwFMu4CGVd1r8a7e15V2/2xu7P7DUPd1F/4EZfe+/XdqPziQ658ypEn5kUp9wErbxJFllOTMk8B1MzMW7HcvPJbUmVm8lux1atH5ZdiD2St+rysKMrWDOXtpV23s6zyzp1J1sNlFx9C3ym34KKHpcovVSkKyTca61hKVqZFWGyZxn68j3yGcSt3CytK698RicZuOKdh+JEUt8ebA/eGdJkVlyens7YbFsUn8tQIiYgeaojxboGKQ7l1m8Lwxaf2y896bj20fNPxoN3l+5D6Ju/r1HK2f3LOrvILXp4yywkVtpVU3s7ib5GgsDZhmX+O9px+TycFcUF0swc1UT91ydWhjQfJV9XGgztX9jce5BfbNx7kV2obD3o8gJUlOn1rT/lCMmZSfrcIbmcjIkLb9jRf4MPXarxt7F1YTdXX9y6UJ2s7ECoIbA+BlRn3qapttNfG0k5UjPhHVdsutBFWY15NWi3OVtsm3lC6+cRzGpdHH4KIptIk1Z6KC/6lQfa8VrGpotxicVbaLi6+4fbhW1jRUhpQ5vN/h+mgeLOn4k2Y4s0/keL5ErGG4uVJTcWbSsWbYMVLV3mAtLbqHxAFJl2jwOQPGwX0nOhgFLB6RgELFgWs3lHgeZBhztr2H54hFG0pFW2BFW19F4o2H1D0H8EhDqrT7qlOG6ZO+3jqDOQfZjwMOZoqtJUqtMEqtJ9ShbO6CG21CK3HEuET8H5QbE5PsTkwsTnfUVWoKS5HKS4HLC7nexCX/WxrOE0xuT3F5MLE5D6TmsuZ83/7RuebXW5NfhVEbDzwpYtQlqtUlgtWlvuUyrLrylILy3mSWusROD+osmlPlU1hKps+UZV1bFVNlaqaglU1/Q5U5R6lujq2imY9VTSDqWj2TOoqc8r/dbH4a8hAx0ypqhlYVbOnUNWDOpo+Se30CCwf1NW8p67mMF3Nn6h2OraO5kodzcE6mj9LHc2OUh8dTTfi8wAdRSPu1ReM/CKBgle+u/P4yedStg4hCnVSleqGJlHLKGzVjnBbBmblyaHCK+zVSgZKcTUveIiWNm015FOMRXDlMCvZ1cGXnDsWybO48vmI+3ROQvml+sPa+hO4QLtKy52UOwq1vF1fq9stnFV+US71eCbt8gZvE+co02mVoVRcoKS66woPsdKmVrZorPgRhC0z2fLqs+twPT6x7dKTX+SpPhS0z2/zS0I95dZkz2xlT/LwbKYwxAfoOwYkca9+NJLfvFeZba+hfdBS9in/18X/NCchijK3GgQVEmpMPGSaw7X3zmy5uOWrV0JwF+Ie0FZBH9vSB5Xaxy93NlPQ9896CZRc8D0anljQ7Z66U/qDlkI5bpOxh2zW5r+bV35t/ba4P2X+XKxIF9YcYEdeh4i5skIf3HtO9xd6P3ZW4+2zPbRSVRwVfiRWvPPV6qwZzj/ZKBeeyyOUqo9Z00g32X4TT+Wcta/maVfC1QK4tkp4IfFL06TMv8NzskFZqtHYqeaW9gxY/kp/+h9QSwMEFAAAAAgAAAAhAFtt/ZMDAQAA8QEAABQAAAB3b3JkL3dlYlNldHRpbmdzLnhtbJXRwUoDMRAG4LvgOyy5t9kWFVm6LYhUvIigPkCazrbBTCbMpK716R1rrUgv9ZZJMh8z/JPZO8bqDVgCpdaMhrWpIHlahrRqzcvzfHBtKikuLV2kBK3ZgpjZ9Pxs0jc9LJ6gFP0plSpJGvStWZeSG2vFrwGdDClD0seOGF3RklcWHb9u8sATZlfCIsRQtnZc11dmz/ApCnVd8HBLfoOQyq7fMkQVKck6ZPnR+lO0nniZmTyI6D4Yvz10IR2Y0cURhMEzCXVlqMvsJ9pR2j6qdyeMv8Dl/4DxAUDf3K8SsVtEjUAnqRQzU82AcgkYPmBOfMPUC7D9unYxUv/4cKeF/RPU9BNQSwMEFAAAAAgAM2tQVmndDRX5BQAASxsAABUAAAB3b3JkL3RoZW1lL3RoZW1lMS54bWztWV2v0zYY/itW7ks+mqQJoqB+wgYHEOeMiUs3cRNznDiK3XNOhZAmuJw0aRqbdjGk3e1i2oYE0m7Yrzkb08Yk/sIcN22T1oGxlQkkWukcfzzP68fva7920nMXThICjlDOME27mnnG0ABKAxriNOpqMz5teRpgHKYhJDRFXW2OmHbh/Dl4lscoQUCwUybKiel0tZjz7Kyus0B0QXYmwUFOGZ3yMwFNdDqd4gDpkpYQ3TJMS08gTrXSBtzi0wylom9K8wRyUc0jPczhsVAm+YZb8lOYCGHXpH1wUNjXVgJHRPxJOSsaApLvF6ZRjSGx4aFZ/GNzNiA5OIKkq4lxQnp8gE64BghkXHR0NUN+NKCfP6evWIQ3kCvEsfwsiSUjPLQkMY8mK6YxsjzbXI8gEYRvA0de8V1blAgYBGK25hbYdFzDs5bgCmpRVFj3O2Z7g1AZob09gu/2LbtOkKhF0d6e6NgfDZ06QaIWRWeL0DOsvt+uEyRqUXS3CPao17FGdYJExQSnh9twt+N57hK+wkwpuaTE+65rdIZL/BqmV1bawkDKm9ZdAm/TfCwAMsqQ4xTweYamMBC4XsYpA0PMMgLnGshgSploNizTFIvQNqzVd+F3eBbBCr1sC9h2WyEJsCDHGe9qHwrDWgXz4ukPL54+Bqf3npze+/n0/v3Tez+paJdgGlVpz7/7/K+Hn4A/H3/7/MGXDQRWJfz246e//vJFA5JXkc++evT7k0fPvv7sj+8fqPC9HE6q+AOcIAauomNwgybF5BRDoEn+mpSDGOIqpZdGDKawIKngIx7X4FfnkEAVsI/qjryZi+ShRF6c3a6J3o/zGccq5OU4qSH3KCV9mqsndlkOV/HFLI0axs9nVeANCI+Uww82Qj2aZWL9Y6XRQYxqUq8TEX0YoRRxUPTRQ4RUvFsY1/y7tzxtwC0M+hCrHXOAJ1zNuoQTEaA5bAh9zUN7N0GfEuUAQ3RUh4ptAonSKCI1b16EMw4TtWqYkCr0CuSxUuj+PA9qjmdcBD1ChIJRiBhTkq7lxazXpMtQJDL1Ctgj86QOzTk+VEKvQEqr0CE9HMQwydS6cRpXwR+wQ7FiIbhOuVoHre+Zoi4CAtPmyN/EiL/mjv8IR7F6sRQ9s1y5RxCt79E5mUK0MK9vJPwEp6/I/v971hdJ9tk3D9+xfN/LsXqLbWb5RuBmbh/QPMTvRmofwll6HRXb531mf5/Z32f2l+zyN5HP1ylcr171pZ2k8d4/xYTs8zlBV5hM/kwcXuFYNMqKJK2eM7JYFJfj1YBRDmUZ5JR/jHm8H8NMjGPKISJW2o4YyCgTJ4jWaFyeP7Nkj4blw5y5eswVDMjXHYaz7hDnFV80u53KU/FqBFmLWFVDwX4dHdXh6jraKh2d9j/UIee3GyG+SohnvlSIXgmPuGsBcbQK39jlywUWQILCImClgWWcdx7zRpfW526ppujbu4t5TUd17dV1VBdlDEO01b7jqPuV2NYkWmolHe/NRF3fThgkrdfAsdiFbUeQA5h1tam4TYpikgmDrLiDQBKJ13sBL/39r9JNljM+hCxe4GRX6YMEc5QDghOx8mvRIOlanmmJLPE26/PFJn8L9emb0UbTKQp4Q8u6KvpKK8ru/4ouKnQmdO/H4TGYkFl+AwpvOR2z8GKIGV+5NMR5ZaGvXbmRw8qdWXtJuN6xkGQxLI+bWppf4GV5pacyESl1c1r1ejmbSTTeybH8atZGJm06W4pTtSGfvLl7QEVXu0GXo05/vvfKA+S/HxUVeV6DvHaDvMZzZZe3hsqA62XaeHzs/JzYXMN65Roqa1u/itDJbbEPhuJ6OyOclW8UTsRrIyFpwStTg2xeJpwTDmY57mp3DKdnDyxn0DI8Z9Sy27bR8pxeu9VznLY5ckxj2LfuCs/In4gWo4/FWy4y38lPR4qffgAWzrnjWmO/7ffdlt/ujVv2sO+1/IHbbw3dQWc4Hg4czx/f1cCRBNu99sB2R17LNQeDlu0ahXzPb3Vsy+rZnZ43snsCXKbHkzKflM5Y+vT831BLAwQUAAAACAAFbmZZ8IgaroYCAAA2CAAAEQAcAHdvcmQvZG9jdW1lbnQueG1sIKIYACigFAAAAAAAAAAAAAAAAAAAAAAAAAAAAKWVS27bMBBAr6Jq3YT6OLYrxAla59MsCgTNomuaoiQiEocgacvu1brokXqFDinLVhIgsOOFRA7JefOThv/+/L28Xjd1sOLaCJCzMD6PwoBLBrmQ5Sxc2uJsGgbGUpnTGiSfhRtuwuuryzbLgS0bLm2AAGmyVrFZWFmrMkIMq3hDzXkjmAYDhT1n0BAoCsE4aUHnJIniyM+UBsaNQWtzKlfUhFtc85YGikvcLEA31KKoS9JQ/bxUZ0hX1IqFqIXdIDsa9xjAGLTMtoiznUNOJesc2g69hj7Ebqdys82At0g0r9EHkKYSah/GR2m4WfWQ1XtBrJq6P9eqeHRaDW40bXHYAw9xP++Umrrz/H1iHB1QEYfYaRziwkubvScNFXJv+EOpGSQ3vjgOkLwGqPK04txrWKo9TZxGe5DPO5b7r49gbYs8DM2c5sxTRdXuD2zjsTnOofhi6xAxm4avByAm8uNIfWgENQec4yDTN9/OmPH1xxgENYeZyW1eHUVK+i+ZOF1qaUUNtpaGZQ+lBE0XNSYbf48Av/DAtZDAF8C9MQdu8FO+DnrzoWv/C8g3blReJ1NU0wfMdTKPJtPbcXpqT3LJ82DL19aDvyTj9DaaeOM6wMeI/OcsHN2l36bzNO3WH3VA3MRefed1DZ+DX6Dr/NMlcUvurV+pT6ZJOr/7Onqt/lIF38qtG87so0eo8uk3UrA7xUkywkuzzbAs8cW0m4MW2MpnoQJtNRU27Liq/EGdcQvYWeNRd1aLssKjvbgAawHvjV6ueTHYrTjNOd5Rk8SLBYAdiOXSehEFb49BbXDZKMqwyP6QX8e7+1674ma1kPxRWIbOp+NuG4Pt48RpV2ec9Pf91X9QSwMEFAAAAAgABW5mWXz5ghLhAAAAQQIAAAsAAABfcmVscy8ucmVsc52SS04DMQyGrxJ53/G0SAihpt100x1CvYCVeGYimocS98HZWHAkrkDoBiLxUpe2f3/6HOXt5XW5Pvu9OnIuLgYN864HxcFE68Ko4SDD7A7Wq+Uj70lqokwuFVVXQtEwiaR7xGIm9lS6mDjUyRCzJ6llHjGReaKRcdH3t5i/MqBlqt1z4v8Q4zA4w5toDp6DfANGPgsHy3aWct3P4riA2lEeWTTYaB5quyCl1FU0qK3VkLf2BhReqfTzkehZyJIQmpj5d6GPRGO0uN7o70dqE586p5gtVqdLu9GZX3Sw+Qird1BLAwQUAAAACAAFbmZZvn2nPeMAAAAmAwAAHAAAAHdvcmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHO1kj1uwzAMha8icK9lpz8oiihZumRNfQFFpmwjtiSITNqcrUOP1CtUcIHWQjN08fgeyfe+gZ/vH+vt2ziIM0bqvVNQFSUIdMY3vWsVnNjePMJ2s97joDltUNcHEunEkYKOOTxJSabDUVPhA7o0sT6OmpOMrQzaHHWLclWWDzLOMyDPFPUl4H8SvbW9wWdvTiM6vhIsX/HwgsyJn0DUOrbICmZmkRJB7BoFcdfcgpCLkdAfDLrGsFqUgS8DzgkmnfVXS/ZzusXf+kl+m1UGcb8khPWOa30YZiA/VkZxN1HI7Ns3X1BLAwQUAAAACAAccmZZUlo+hkwBAAAaBQAAEwAAAFtDb250ZW50X1R5cGVzXS54bWy1lE1OwzAQha9ieVslblkghJp2AWyhEr2A60xSC8e27Onf2VhwJK7AJGkjhEqDaLuJlMy8972xMv58/xhPt5VhawhRO5vxUTrkDKxyubZlxldYJHd8OhnPdx4io1YbM75E9PdCRLWESsbUebBUKVyoJNJrKIWX6k2WIG6Gw1uhnEWwmGDtwSfjRyjkyiB72tLnFhvARM4e2saalXHpvdFKItXF2uY/KMmekJKy6YlL7eOAGjgTRxFN6VfCQfhCJxF0DmwmAz7LitrExoVc5E6tKpKmp32OJHVFoRV0+trNB6cgRjriyqRdpZLaDnqDRNwZiJeP0fr+gQ+IpLhGgr1zf4YNLF6vFuObeX+SgsBzuTBw+RyddX8KpEWE9jk6O0hjc5JJrbPgfKTNDv8Y/LC6tTqhkT0E1D2/Xock77MnhPpWyCE/BhfNTTf5AlBLAQIUABQAAAAIAK+YSUqNEzDqOgEAAKcCAAAQAAAAAAAAAAAAAAAAAAAAAABkb2NQcm9wcy9hcHAueG1sUEsBAhQAFAAAAAgABW5mWVDfgmNYAQAAjAIAABEAAAAAAAAAAAAAAAAAaAEAAGRvY1Byb3BzL2NvcmUueG1sUEsBAhQAFAAAAAgABW5mWZ5WdX44AgAAAgkAABIAAAAAAAAAAAAAAAAA7wIAAHdvcmQvZm9udFRhYmxlLnhtbFBLAQIUABQAAAAIAAVuZlnBTYlGHgQAAA4MAAARAAAAAAAAAAAAAAAAAFcFAAB3b3JkL3NldHRpbmdzLnhtbFBLAQIUABQAAAAIAAVuZll/jXz0SQ4AAOeiAAAPAAAAAAAAAAAAAAAAAKQJAAB3b3JkL3N0eWxlcy54bWxQSwECFAAUAAAACAAAACEAW239kwMBAADxAQAAFAAAAAAAAAAAAAAAAAAaGAAAd29yZC93ZWJTZXR0aW5ncy54bWxQSwECFAAUAAAACAAza1BWad0NFfkFAABLGwAAFQAAAAAAAAAAAAAAAABPGQAAd29yZC90aGVtZS90aGVtZTEueG1sUEsBAhQAFAAAAAgABW5mWfCIGq6GAgAANggAABEAAAAAAAAAAAAAAAAAex8AAHdvcmQvZG9jdW1lbnQueG1sUEsBAhQAFAAAAAgABW5mWXz5ghLhAAAAQQIAAAsAAAAAAAAAAAAAAAAATCIAAF9yZWxzLy5yZWxzUEsBAhQAFAAAAAgABW5mWb59pz3jAAAAJgMAABwAAAAAAAAAAAAAAAAAViMAAHdvcmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHNQSwECFAAUAAAACAAccmZZUlo+hkwBAAAaBQAAEwAAAAAAAAAAAAAAAABzJAAAW0NvbnRlbnRfVHlwZXNdLnhtbFBLBQYAAAAACwALAMECAADwJQAAAAA=" - ) - - expected_result_pdf = "Hello, World!" - expected_result_docx = "Hello, World!!" - - async def setup_after_prep(self, module_test): - module_test.set_expect_requests( - {"uri": "/"}, - {"response_data": ''}, - ) - module_test.set_expect_requests( - {"uri": "/Test_PDF"}, - {"response_data": self.pdf_data, "headers": {"Content-Type": "application/pdf"}}, - ) - module_test.set_expect_requests( - {"uri": "/Test_DOCX"}, - { - "response_data": self.docx_data, - "headers": {"Content-Type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - }, - ) - - def check(self, module_test, events): - filesystem_events = [e for e in events if e.type == "FILESYSTEM"] - assert 2 == len(filesystem_events), filesystem_events - for filesystem_event in filesystem_events: - file = Path(filesystem_event.data["path"]) - assert file.is_file(), "Destination file doesn't exist" - assert open(file, "rb").read() == self.pdf_data or open(file, "rb").read() == self.docx_data, ( - f"File at {file} does not contain the correct content" - ) - raw_text_events = [e for e in events if e.type == "RAW_TEXT"] - assert 2 == len(raw_text_events), "Failed to emit RAW_TEXT event" - for raw_text_event in raw_text_events: - assert raw_text_event.data in [ - self.expected_result_pdf, - self.expected_result_docx, - ], f"Text extracted from {raw_text_event.data['path']} is incorrect, got {raw_text_event.data}" diff --git a/bbot/test/test_step_2/module_tests/test_module_kreuzberg.py b/bbot/test/test_step_2/module_tests/test_module_kreuzberg.py new file mode 100644 index 0000000000..217375dbd5 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_kreuzberg.py @@ -0,0 +1,89 @@ +import base64 +from pathlib import Path +from .base import ModuleTestBase + +from ...bbot_fixtures import * + + +class TestKreuzberg(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["kreuzberg", "filedownload", "httpx", "excavate", "speculate"] + config_overrides = { + "web": { + "spider_distance": 2, + "spider_depth": 2, + }, + "modules": { + "filedownload": { + "output_folder": bbot_test_dir / "filedownload", + }, + }, + } + + pdf_data = base64.b64decode( + "JVBERi0xLjMKJe+/ve+/ve+/ve+/vSBSZXBvcnRMYWIgR2VuZXJhdGVkIFBERiBkb2N1bWVudCBodHRwOi8vd3d3LnJlcG9ydGxhYi5jb20KMSAwIG9iago8PAovRjEgMiAwIFIKPj4KZW5kb2JqCjIgMCBvYmoKPDwKL0Jhc2VGb250IC9IZWx2ZXRpY2EgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YxIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKMyAwIG9iago8PAovQ29udGVudHMgNyAwIFIgL01lZGlhQm94IFsgMCAwIDU5NS4yNzU2IDg0MS44ODk4IF0gL1BhcmVudCA2IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iago0IDAgb2JqCjw8Ci9QYWdlTW9kZSAvVXNlTm9uZSAvUGFnZXMgNiAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjUgMCBvYmoKPDwKL0F1dGhvciAoYW5vbnltb3VzKSAvQ3JlYXRpb25EYXRlIChEOjIwMjQwNjAzMTg1ODE2KzAwJzAwJykgL0NyZWF0b3IgKFJlcG9ydExhYiBQREYgTGlicmFyeSAtIHd3dy5yZXBvcnRsYWIuY29tKSAvS2V5d29yZHMgKCkgL01vZERhdGUgKEQ6MjAyNDA2MDMxODU4MTYrMDAnMDAnKSAvUHJvZHVjZXIgKFJlcG9ydExhYiBQREYgTGlicmFyeSAtIHd3dy5yZXBvcnRsYWIuY29tKSAKICAvU3ViamVjdCAodW5zcGVjaWZpZWQpIC9UaXRsZSAodW50aXRsZWQpIC9UcmFwcGVkIC9GYWxzZQo+PgplbmRvYmoKNiAwIG9iago8PAovQ291bnQgMSAvS2lkcyBbIDMgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iago3IDAgb2JqCjw8Ci9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvTGVuZ3RoIDEwNwo+PgpzdHJlYW0KR2FwUWgwRT1GLDBVXEgzVFxwTllUXlFLaz90Yz5JUCw7VyNVMV4yM2loUEVNXz9DVzRLSVNpOTBNakdeMixGUyM8UkM1K2MsbilaOyRiSyRiIjVJWzwhXlREI2dpXSY9NVgsWzVAWUBWfj5lbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA4CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA3MyAwMDAwMCBuIAowMDAwMDAwMTA0IDAwMDAwIG4gCjAwMDAwMDAyMTEgMDAwMDAgbiAKMDAwMDAwMDQxNCAwMDAwMCBuIAowMDAwMDAwNDgyIDAwMDAwIG4gCjAwMDAwMDA3NzggMDAwMDAgbiAKMDAwMDAwMDgzNyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzw4MGQ5ZjViOTY0ZmM5OTI4NDUwMWRlYjdhNmE2MzdmNz48ODBkOWY1Yjk2NGZjOTkyODQ1MDFkZWI3YTZhNjM3Zjc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAoaHR0cDovL3d3dy5yZXBvcnRsYWIuY29tKQoKL0luZm8gNSAwIFIKL1Jvb3QgNCAwIFIKL1NpemUgOAo+PgpzdGFydHhyZWYKMTAzNAolJUVPRg==" + ) + + docx_data = base64.b64decode( + "UEsDBBQAAAAAAFitWVzXeYTquAEAALgBAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbDw/eG1sIHZl" + "cnNpb249IjEuMCIgZW5jb2Rpbmc9IlVURi04IiBzdGFuZGFsb25lPSJ5ZXMiPz4KPFR5cGVzIHht" + "bG5zPSJodHRwOi8vc2NoZW1hcy5vcGVueG1sZm9ybWF0cy5vcmcvcGFja2FnZS8yMDA2L2NvbnRl" + "bnQtdHlwZXMiPgogIDxEZWZhdWx0IEV4dGVuc2lvbj0icmVscyIgQ29udGVudFR5cGU9ImFwcGxp" + "Y2F0aW9uL3ZuZC5vcGVueG1sZm9ybWF0cy1wYWNrYWdlLnJlbGF0aW9uc2hpcHMreG1sIi8+CiAg" + "PERlZmF1bHQgRXh0ZW5zaW9uPSJ4bWwiIENvbnRlbnRUeXBlPSJhcHBsaWNhdGlvbi94bWwiLz4K" + "ICA8T3ZlcnJpZGUgUGFydE5hbWU9Ii93b3JkL2RvY3VtZW50LnhtbCIgQ29udGVudFR5cGU9ImFw" + "cGxpY2F0aW9uL3ZuZC5vcGVueG1sZm9ybWF0cy1vZmZpY2Vkb2N1bWVudC53b3JkcHJvY2Vzc2lu" + "Z21sLmRvY3VtZW50Lm1haW4reG1sIi8+CjwvVHlwZXM+UEsDBBQAAAAAAFitWVwgG4bqLgEAAC4B" + "AAALAAAAX3JlbHMvLnJlbHM8P3htbCB2ZXJzaW9uPSIxLjAiIGVuY29kaW5nPSJVVEYtOCIgc3Rh" + "bmRhbG9uZT0ieWVzIj8+CjxSZWxhdGlvbnNoaXBzIHhtbG5zPSJodHRwOi8vc2NoZW1hcy5vcGVu" + "eG1sZm9ybWF0cy5vcmcvcGFja2FnZS8yMDA2L3JlbGF0aW9uc2hpcHMiPgogIDxSZWxhdGlvbnNo" + "aXAgSWQ9InJJZDEiIFR5cGU9Imh0dHA6Ly9zY2hlbWFzLm9wZW54bWxmb3JtYXRzLm9yZy9vZmZp" + "Y2VEb2N1bWVudC8yMDA2L3JlbGF0aW9uc2hpcHMvb2ZmaWNlRG9jdW1lbnQiIFRhcmdldD0id29y" + "ZC9kb2N1bWVudC54bWwiLz4KPC9SZWxhdGlvbnNoaXBzPlBLAwQUAAAAAABYrVlcNNIntwABAAAA" + "AQAAEQAAAHdvcmQvZG9jdW1lbnQueG1sPD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRG" + "LTgiIHN0YW5kYWxvbmU9InllcyI/Pgo8dzpkb2N1bWVudCB4bWxuczp3PSJodHRwOi8vc2NoZW1h" + "cy5vcGVueG1sZm9ybWF0cy5vcmcvd29yZHByb2Nlc3NpbmdtbC8yMDA2L21haW4iPgogIDx3OmJv" + "ZHk+CiAgICA8dzpwPgogICAgICA8dzpyPgogICAgICAgIDx3OnQ+SGVsbG8sIFdvcmxkISE8L3c6" + "dD4KICAgICAgPC93OnI+CiAgICA8L3c6cD4KICA8L3c6Ym9keT4KPC93OmRvY3VtZW50PlBLAQIU" + "AxQAAAAAAFitWVzXeYTquAEAALgBAAATAAAAAAAAAAAAAACAAQAAAABbQ29udGVudF9UeXBlc10u" + "eG1sUEsBAhQDFAAAAAAAWK1ZXCAbhuouAQAALgEAAAsAAAAAAAAAAAAAAIAB6QEAAF9yZWxzLy5y" + "ZWxzUEsBAhQDFAAAAAAAWK1ZXDTSJ7cAAQAAAAEAABEAAAAAAAAAAAAAAIABQAMAAHdvcmQvZG9j" + "dW1lbnQueG1sUEsFBgAAAAADAAMAuQAAAG8EAAAAAA==" + ) + + expected_result_pdf = "Hello, World!" + expected_result_docx = "Hello, World!!" + + async def setup_after_prep(self, module_test): + module_test.set_expect_requests( + {"uri": "/"}, + {"response_data": ''}, + ) + module_test.set_expect_requests( + {"uri": "/Test_PDF"}, + {"response_data": self.pdf_data, "headers": {"Content-Type": "application/pdf"}}, + ) + module_test.set_expect_requests( + {"uri": "/Test_DOCX"}, + { + "response_data": self.docx_data, + "headers": {"Content-Type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + }, + ) + + def check(self, module_test, events): + filesystem_events = [e for e in events if e.type == "FILESYSTEM"] + assert 2 == len(filesystem_events), filesystem_events + for filesystem_event in filesystem_events: + file = Path(filesystem_event.data["path"]) + assert file.is_file(), "Destination file doesn't exist" + assert open(file, "rb").read() == self.pdf_data or open(file, "rb").read() == self.docx_data, ( + f"File at {file} does not contain the correct content" + ) + raw_text_events = [e for e in events if e.type == "RAW_TEXT"] + assert 2 == len(raw_text_events), "Failed to emit RAW_TEXT event" + for raw_text_event in raw_text_events: + assert raw_text_event.data in [ + self.expected_result_pdf, + self.expected_result_docx, + ], f"Text extracted from {raw_text_event.data['path']} is incorrect, got {raw_text_event.data}" diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py index 2c3795cabe..57acc74656 100644 --- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py +++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py @@ -1309,7 +1309,7 @@ def check(self, module_test, events): class TestTrufflehog_RAWText(ModuleTestBase): targets = ["http://127.0.0.1:8888/test.pdf"] - modules_overrides = ["httpx", "trufflehog", "filedownload", "extractous"] + modules_overrides = ["httpx", "trufflehog", "filedownload", "kreuzberg"] download_dir = bbot_test_dir / "test_trufflehog_rawtext" config_overrides = { diff --git a/docs/data/chord_graph/entities.json b/docs/data/chord_graph/entities.json index 5afb9f73aa..f00a513a22 100644 --- a/docs/data/chord_graph/entities.json +++ b/docs/data/chord_graph/entities.json @@ -1250,7 +1250,7 @@ }, { "id": 75, - "name": "extractous", + "name": "kreuzberg", "parent": 99999999, "consumes": [ 10 diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 4205db8810..50af1841f6 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -83,7 +83,7 @@ | docker_pull | scan | No | Download images from a docker repository | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | | dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 | | emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | -| extractous | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | +| kreuzberg | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | | fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | | git_clone | scan | No | Clone code github repositories | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | | gitdumper | scan | No | Download a leaked .git folder recursively or by fuzzing common names | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2025-02-11 | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 6562a5ca69..b60c3260e4 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -72,7 +72,7 @@ Presets: Modules: -m, --modules MODULE [MODULE ...] - Modules to enable. Choices: affiliates,ajaxpro,anubisdb,apkpure,asn,aspnet_bin_exposure,azure_realm,azure_tenant,baddns,baddns_direct,baddns_zone,badsecrets,bevigil,bucket_amazon,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,bucket_microsoft,bufferoverrun,builtwith,bypass403,c99,censys_dns,censys_ip,certspotter,chaos,code_repository,credshed,crt,crt_db,dehashed,digitorus,dnsbimi,dnsbrute,dnsbrute_mutations,dnscaa,dnscommonsrv,dnsdumpster,dnstlsrpt,docker_pull,dockerhub,dotnetnuke,emailformat,extractous,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,gitdumper,github_codesearch,github_org,github_usersearch,github_workflows,gitlab_com,gitlab_onprem,google_playstore,gowitness,graphql_introspection,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,jadx,leakix,legba,lightfuzz,medusa,myssl,newsletters,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,portfilter,portscan,postman,postman_download,rapiddns,reflected_parameters,retirejs,robots,securitytrails,securitytxt,shodan_dns,shodan_idb,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,subdomainradar,telerik,trickest,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wayback,wpscan + Modules to enable. Choices: affiliates,ajaxpro,anubisdb,apkpure,asn,aspnet_bin_exposure,azure_realm,azure_tenant,baddns,baddns_direct,baddns_zone,badsecrets,bevigil,bucket_amazon,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,bucket_microsoft,bufferoverrun,builtwith,bypass403,c99,censys_dns,censys_ip,certspotter,chaos,code_repository,credshed,crt,crt_db,dehashed,digitorus,dnsbimi,dnsbrute,dnsbrute_mutations,dnscaa,dnscommonsrv,dnsdumpster,dnstlsrpt,docker_pull,dockerhub,dotnetnuke,emailformat,kreuzberg,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,gitdumper,github_codesearch,github_org,github_usersearch,github_workflows,gitlab_com,gitlab_onprem,google_playstore,gowitness,graphql_introspection,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,jadx,leakix,legba,lightfuzz,medusa,myssl,newsletters,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,portfilter,portscan,postman,postman_download,rapiddns,reflected_parameters,retirejs,robots,securitytrails,securitytxt,shodan_dns,shodan_idb,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,subdomainradar,telerik,trickest,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wayback,wpscan -l, --list-modules List available modules. -lmo, --list-module-options Show all module config options diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index f518aa8f02..548f60b61a 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -530,7 +530,7 @@ In addition to the stated options for each module, the following universal optio | modules.dnstlsrpt.emit_urls | bool | Emit URL_UNVERIFIED events | True | | modules.docker_pull.all_tags | bool | Download all tags from each registry (Default False) | False | | modules.docker_pull.output_folder | str | Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage. | | -| modules.extractous.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'json', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'rsa', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | +| modules.kreuzberg.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'json', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'rsa', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | | modules.fullhunt.api_key | str | FullHunt API Key | | | modules.git_clone.api_key | str | Github token | | | modules.git_clone.output_folder | str | Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage. | | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index f554b8b797..7d168e4456 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -113,7 +113,7 @@ Below is a full list of event types along with which modules produce/consume the | DNS_NAME | 60 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, emailformat, fullhunt, github_codesearch, github_usersearch, hackertarget, hunterio, leakix, myssl, nmap_xml, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback | anubisdb, azure_tenant, bevigil, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, crt, crt_db, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnsresolve, fullhunt, hackertarget, hunterio, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, shodan_idb, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback | | DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | | EMAIL_ADDRESS | 1 | 11 | emails | credshed, dehashed, dnscaa, dnstlsrpt, emailformat, github_usersearch, hunterio, pgp, securitytxt, skymem, sslcert | -| FILESYSTEM | 4 | 9 | extractous, jadx, trufflehog, unarchive | apkpure, docker_pull, filedownload, git_clone, gitdumper, github_workflows, jadx, postman_download, unarchive | +| FILESYSTEM | 4 | 9 | kreuzberg, jadx, trufflehog, unarchive | apkpure, docker_pull, filedownload, git_clone, gitdumper, github_workflows, jadx, postman_download, unarchive | | FINDING | 2 | 32 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bypass403, git, gitlab_onprem, graphql_introspection, host_header, hunt, legba, lightfuzz, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, reflected_parameters, retirejs, shodan_idb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | | GEOLOCATION | 0 | 2 | | ip2location, ipstack | | HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | @@ -127,7 +127,7 @@ Below is a full list of event types along with which modules produce/consume the | PASSWORD | 0 | 2 | | credshed, dehashed | | PROTOCOL | 3 | 2 | legba, medusa, nmap_xml | censys_ip, fingerprintx | | RAW_DNS_RECORD | 0 | 3 | | dnsbimi, dnsresolve, dnstlsrpt | -| RAW_TEXT | 2 | 1 | excavate, trufflehog | extractous | +| RAW_TEXT | 2 | 1 | excavate, trufflehog | kreuzberg | | SOCIAL | 7 | 4 | dockerhub, github_org, gitlab_com, gitlab_onprem, gowitness, postman, speculate | dockerhub, github_usersearch, gitlab_onprem, social | | STORAGE_BUCKET | 8 | 5 | baddns_direct, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, speculate | bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft | | TECHNOLOGY | 4 | 8 | asset_inventory, gitlab_onprem, web_report, wpscan | badsecrets, censys_ip, dotnetnuke, gitlab_onprem, gowitness, nuclei, shodan_idb, wpscan | diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 0287ecdec6..18358ae0ce 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -114,8 +114,8 @@ A single module can have multiple flags. For example, the `securitytrails` modul | Flag | # Modules | Description | Modules | |------------------|-------------|----------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 98 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, apkpure, asn, aspnet_bin_exposure, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, extractous, filedownload, fingerprintx, fullhunt, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, gowitness, graphql_introspection, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, jadx, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portfilter, portscan, postman, postman_download, rapiddns, reflected_parameters, retirejs, robots, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback | -| passive | 70 | Never connects to target systems | affiliates, aggregate, anubisdb, apkpure, asn, azure_realm, azure_tenant, bevigil, bucket_file_enum, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, excavate, extractous, fullhunt, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, google_playstore, hackertarget, hunterio, ip2location, ipneighbor, ipstack, jadx, leakix, myssl, otx, passivetotal, pgp, portfilter, postman, postman_download, rapiddns, securitytrails, shodan_dns, shodan_idb, sitedossier, skymem, social, speculate, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback | +| safe | 98 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, apkpure, asn, aspnet_bin_exposure, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, kreuzberg, filedownload, fingerprintx, fullhunt, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, gowitness, graphql_introspection, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, jadx, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portfilter, portscan, postman, postman_download, rapiddns, reflected_parameters, retirejs, robots, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback | +| passive | 70 | Never connects to target systems | affiliates, aggregate, anubisdb, apkpure, asn, azure_realm, azure_tenant, bevigil, bucket_file_enum, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, excavate, kreuzberg, fullhunt, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, google_playstore, hackertarget, hunterio, ip2location, ipneighbor, ipstack, jadx, leakix, myssl, otx, passivetotal, pgp, portfilter, postman, postman_download, rapiddns, securitytrails, shodan_dns, shodan_idb, sitedossier, skymem, social, speculate, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback | | active | 52 | Makes active connections to target systems | ajaxpro, aspnet_bin_exposure, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bypass403, dnsbrute, dnsbrute_mutations, dnscommonsrv, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab_com, gitlab_onprem, gowitness, graphql_introspection, host_header, httpx, hunt, iis_shortnames, legba, lightfuzz, medusa, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, reflected_parameters, retirejs, robots, securitytxt, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wpscan | | subdomain-enum | 51 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_direct, baddns_zone, bevigil, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, crt, crt_db, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, sslcert, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, virustotal, wayback | | aggressive | 22 | Generates a large amount of network traffic | bypass403, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, legba, lightfuzz, medusa, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan | diff --git a/pyproject.toml b/pyproject.toml index 62fa5cc6e7..35452f70b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] description = "OSINT automation for hackers." readme = "README.md" license = "AGPL-3.0" -requires-python = ">=3.10" +requires-python = ">=3.10,<3.15" authors = [ { name = "TheTechromancer" }, { name = "Paul Mueller" }, @@ -41,10 +41,10 @@ dependencies = [ "pyzmq>=26.0.3,<28.0.0", "httpx>=0.28.1,<1", "puremagic>=1.28,<2", - "pydantic>=2.9.2,<3", + "pydantic>=2.12.2,<3", "radixtarget>=3.0.13,<4", "orjson>=3.10.12,<4", - "ansible-core>=2.15.13,<3", + "ansible-core>=2.17,<3", "tldextract>=5.3.0,<6", "cloudcheck>=9.2.0,<10", ] diff --git a/uv.lock b/uv.lock index 18cf64a3dc..dfd4409fc1 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.10" +requires-python = ">=3.10, <3.15" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", @@ -222,7 +222,7 @@ docs = [ [package.metadata] requires-dist = [ - { name = "ansible-core", specifier = ">=2.15.13,<3" }, + { name = "ansible-core", specifier = ">=2.17,<3" }, { name = "ansible-runner", specifier = ">=2.3.2,<3" }, { name = "beautifulsoup4", specifier = ">=4.12.2,<5" }, { name = "cachetools", specifier = ">=5.3.2,<7.0.0" }, @@ -239,7 +239,7 @@ requires-dist = [ { name = "psutil", specifier = ">=5.9.4,<8.0.0" }, { name = "puremagic", specifier = ">=1.28,<2" }, { name = "pycryptodome", specifier = ">=3.17,<4" }, - { name = "pydantic", specifier = ">=2.9.2,<3" }, + { name = "pydantic", specifier = ">=2.12.2,<3" }, { name = "pyjwt", specifier = ">=2.7.0,<3" }, { name = "pyzmq", specifier = ">=26.0.3,<28.0.0" }, { name = "radixtarget", specifier = ">=3.0.13,<4" },