Support comments in target lists

bbot/core/helpers/misc.py

@@ -1118,6 +1118,32 @@ def str_or_file(s):
         yield s
 
 
+_comment_re = re.compile(r"\s#")
+
+
+def strip_comments(line):
+    """Strip #-style comments from a line.
+
+    Handles full-line comments (``# ...``) and inline comments (``target # ...``).
+    The ``#`` must be preceded by whitespace to count as an inline comment,
+    so URL fragments like ``http://example.com/page#section`` are preserved.
+
+    Examples:
+        >>> strip_comments("evilcorp.com # main domain")
+        'evilcorp.com'
+        >>> strip_comments("# full line comment")
+        ''
+        >>> strip_comments("http://example.com/page#section")
+        'http://example.com/page#section'
+    """
+    if line.lstrip().startswith("#"):
+        return ""
+    m = _comment_re.search(line)
+    if m:
+        return line[: m.start()]
+    return line
+
+
 split_regex = re.compile(r"[\s,]")
 
 
@@ -1128,6 +1154,7 @@ def chain_lists(
     remove_blank=True,
     validate=False,
     validate_chars='<>:"/\\|?*)',
+    _strip_comments=False,
 ):
     """Chains together list elements, allowing for entries separated by commas.
 
@@ -1143,6 +1170,7 @@ def chain_lists(
         remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True.
         validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False.
         validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)'
+        _strip_comments (bool, optional): Whether to strip ``#``-style comments from entries and file lines. Defaults to False.
 
     Returns:
         list: The list of chained elements.
@@ -1172,6 +1200,8 @@ def chain_lists(
                     new_msg = str(msg).format(filename=f_path)
                     log.info(new_msg)
                 for line in str_or_file(f):
+                    if _strip_comments:
+                        line = strip_comments(line)
                     final_list[line] = None
             else:
                 final_list[f] = None

bbot/scanner/preset/args.py

@@ -403,14 +403,14 @@ def sanitize_args(self):
         self.parsed.exclude_modules = chain_lists(self.parsed.exclude_modules)
         self.parsed.output_modules = chain_lists(self.parsed.output_modules)
         self.parsed.targets = chain_lists(
-            self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}"
+            self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}", _strip_comments=True
         )
         if self.parsed.seeds is not None:
             self.parsed.seeds = chain_lists(
-                self.parsed.seeds, try_files=True, msg="Reading seeds from file: {filename}"
+                self.parsed.seeds, try_files=True, msg="Reading seeds from file: {filename}", _strip_comments=True
             )
         self.parsed.blacklist = chain_lists(
-            self.parsed.blacklist, try_files=True, msg="Reading blacklist from file: {filename}"
+            self.parsed.blacklist, try_files=True, msg="Reading blacklist from file: {filename}", _strip_comments=True
         )
         self.parsed.flags = chain_lists(self.parsed.flags)
         self.parsed.exclude_flags = chain_lists(self.parsed.exclude_flags)

bbot/scanner/preset/preset.py

@@ -673,20 +673,23 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
             cls._resolve_file_entries(target_vals),
             try_files=True,
             msg="Reading targets from preset file: {filename}",
+            _strip_comments=True,
         )
         seeds = preset_dict.get("seeds")
         if seeds is not None:
             seeds = chain_lists(
                 cls._resolve_file_entries(seeds),
                 try_files=True,
                 msg="Reading seeds from preset file: {filename}",
+                _strip_comments=True,
             )
         blacklist = preset_dict.get("blacklist")
         if blacklist is not None:
             blacklist = chain_lists(
                 cls._resolve_file_entries(blacklist),
                 try_files=True,
                 msg="Reading blacklist from preset file: {filename}",
+                _strip_comments=True,
             )
         new_preset = cls(
             *targets,

bbot/test/test_step_1/test_target.py

@@ -710,3 +710,65 @@ def test_target_pickle():
 
     # hashes match
     assert target.hash == restored.hash
+
+
+def test_target_comments_from_file(tmp_path):
+    """Comments in a target file are stripped when loaded via chain_lists."""
+    from bbot.core.helpers.misc import chain_lists
+
+    target_file = tmp_path / "targets.txt"
+    target_file.write_text(
+        "# My target list\n"
+        "evilcorp.com # main domain\n"
+        "\n"
+        "  # another comment\n"
+        "othercorp.com\n"
+        "192.168.1.0/24 # lab network\n"
+        "http://example.com/page#fragment # with a URL fragment\n"
+    )
+
+    result = chain_lists([str(target_file)], try_files=True, _strip_comments=True)
+    assert "evilcorp.com" in result
+    assert "othercorp.com" in result
+    assert "192.168.1.0/24" in result
+    assert "http://example.com/page#fragment" in result
+    # comments and blank lines are gone
+    assert not any(r.lstrip().startswith("#") for r in result)
+    assert len(result) == 4
+
+
+def test_target_comments_blacklist_file(tmp_path):
+    """Comments in a blacklist file are stripped when loaded via chain_lists."""
+    from bbot.core.helpers.misc import chain_lists
+
+    blacklist_file = tmp_path / "blacklist.txt"
+    blacklist_file.write_text("# don't scan the blog\nblog.evilcorp.com # unstable host\n")
+
+    result = chain_lists([str(blacklist_file)], try_files=True, _strip_comments=True)
+    assert result == ["blog.evilcorp.com"]
+
+
+def test_strip_comments_helper():
+    """Unit tests for the strip_comments function."""
+    from bbot.core.helpers.misc import strip_comments
+
+    # full-line comments
+    assert strip_comments("# comment") == ""
+    assert strip_comments("  # indented comment") == ""
+
+    # inline comments
+    assert strip_comments("evilcorp.com # main domain") == "evilcorp.com"
+    assert strip_comments("1.2.3.0/24\t# tab comment") == "1.2.3.0/24"
+
+    # no comment
+    assert strip_comments("evilcorp.com") == "evilcorp.com"
+
+    # URL fragment (no space before #) is preserved
+    assert strip_comments("http://example.com/page#section") == "http://example.com/page#section"
+
+    # URL fragment with trailing inline comment
+    assert strip_comments("http://example.com/page#section # a comment") == "http://example.com/page#section"
+
+    # empty / whitespace
+    assert strip_comments("") == ""
+    assert strip_comments("   ") == "   "