diff --git a/bbot/cli.py b/bbot/cli.py index 2ebe9046e3..1687966594 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -68,6 +68,13 @@ async def _main(): # that don't construct a full Scanner. preset.apply_log_level(apply_core=True) + # which generated config files the user is regenerating this run + reset_labels = [ + label + for label, requested in (("config", options.reset_config), ("secrets", options.reset_secrets)) + if requested + ] + # print help if no arguments if len(sys.argv) == 1: print(preset.args.parser.format_help()) @@ -80,6 +87,41 @@ async def _main(): sys.exit(0) return + # --reset-config / --reset-secrets + if reset_labels: + reset_paths = [ + spec["path"] + for spec in preset.module_loader._generated_config_files() + if spec["label"] in reset_labels + ] + log.hugewarning( + "Regenerating from current defaults. Any settings you have customized " + "(uncommented) in these files WILL BE WIPED OUT:" + ) + for p in reset_paths: + log.warning(f" {p}") + log.warning("A backup of each existing file will be saved with a .bak extension.") + try: + stdin_is_tty = sys.stdin.isatty() + except (ValueError, io.UnsupportedOperation): + stdin_is_tty = False + if not options.yes: + if not stdin_is_tty: + log.error("Refusing to reset config without confirmation; re-run with --yes to proceed.") + sys.exit(1) + return + answer = input("Continue? [y/N] ").strip().lower() + if answer not in ("y", "yes"): + log.info("Aborted. No changes made.") + sys.exit(0) + return + backups = preset.module_loader.reset_config_files(reset_labels) + log.success("Regenerated config files from current defaults.") + for b in backups: + log.info(f"Backup saved: {b}") + sys.exit(0) + return + # --list-presets if options.list_presets: print("") @@ -157,7 +199,32 @@ async def _main(): print(row) return - preset.validate() + try: + preset.validate() + except ValidationError as e: + log.error(str(e)) + # if a bad option actually lives in one of the user's generated + # config files (vs, say, a -c CLI typo), point them at the matching + # reset flag -- validate each file's own contents to be sure + import yaml + from bbot.scanner.preset.validate import validate_preset + + for spec in preset.module_loader._generated_config_files(): + if not spec["path"].exists(): + continue + try: + file_config = yaml.safe_load(spec["path"].read_text()) or {} + except yaml.YAMLError: + continue + if isinstance(file_config, dict) and validate_preset( + {"config": file_config}, module_loader=preset.module_loader + ): + log.warning( + f"Some options in {spec['path']} are not recognized. They may be left over from an " + f"older version of BBOT. You have the option of regenerating from current defaults " + f"with: bbot {spec['reset_flag']}" + ) + return baked_preset = preset.bake() # --current-preset / --current-preset-full diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 7df234ad1b..bf04591132 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -1,5 +1,9 @@ +import os import sys import yaml +import atexit +import shutil +import tempfile from pathlib import Path from .merge import deep_merge @@ -9,15 +13,39 @@ bbot_code_dir = Path(__file__).parent.parent.parent +# cached per-process so every BBOTConfigFiles in a run resolves to the same dir +_test_config_dir = None + + +def isolated_test_config_dir(): + """A throwaway config dir for tests, so we never read or write the user's + real ~/.config/bbot. It's created fresh per run (so previous or concurrent + runs can't interfere) and shared across the run's processes via an env var + so spawned children resolve to the same dir.""" + global _test_config_dir + if _test_config_dir is None: + env_dir = os.environ.get("BBOT_TEST_CONFIG_DIR") + if env_dir: + _test_config_dir = Path(env_dir) + else: + _test_config_dir = Path(tempfile.mkdtemp(prefix="bbot_test_config_")) + os.environ["BBOT_TEST_CONFIG_DIR"] = str(_test_config_dir) + atexit.register(lambda: shutil.rmtree(_test_config_dir, ignore_errors=True)) + return _test_config_dir + class BBOTConfigFiles: - config_dir = (Path.home() / ".config" / "bbot").resolve() defaults_filename = (bbot_code_dir / "defaults.yml").resolve() - config_filename = (config_dir / "bbot.yml").resolve() - secrets_filename = (config_dir / "secrets.yml").resolve() def __init__(self, core): self.core = core + if os.environ.get("BBOT_TESTING", "") == "True": + base_dir = isolated_test_config_dir() + else: + base_dir = Path.home() / ".config" / "bbot" + self.config_dir = base_dir.resolve() + self.config_filename = (self.config_dir / "bbot.yml").resolve() + self.secrets_filename = (self.config_dir / "secrets.yml").resolve() def _get_config(self, filename, name="config") -> dict: filename = Path(filename).resolve() diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 233fc87608..cee81ef6f2 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -1,10 +1,14 @@ +import os import re import ast import sys +import stat import yaml import atexit +import shutil import pickle import logging +import tempfile import importlib import traceback from copy import copy @@ -1059,35 +1063,120 @@ def filter_modules(self, modules=None, mod_type=None): module_list.sort(key=lambda x: x[-1]["type"], reverse=True) return module_list - def ensure_config_files(self): + def _generated_config_files(self): + """The config files BBOT generates, each paired with the content it + should currently hold and the CLI flag that regenerates it. + + Creation and reset both iterate this list, so the two files stay fully + independent: a `secrets.yml` full of API keys is never touched just + because `bbot.yml`'s options changed, and vice versa. + """ files = self.core.files_config - mkdir(files.config_dir) + config_obj = dict(self.core.default_config) + return [ + { + "label": "config", + "path": files.config_filename, + "content": self.core.no_secrets_config(config_obj), + "secret": False, + "reset_flag": "--reset-config", + }, + { + "label": "secrets", + "path": files.secrets_filename, + "content": self.core.secrets_only_config(config_obj), + "secret": True, + "reset_flag": "--reset-secrets", + }, + ] - comment_notice = ( - "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" - + "# Please be sure to uncomment when inserting API keys, etc.\n" - ) + def ensure_config_files(self): + """Create any of the user's generated config files that are missing. - config_obj = dict(self.core.default_config) + Each file is a fully-commented snapshot of the current defaults, + written once and never overwritten. + """ + mkdir(self.core.files_config.config_dir) + for spec in self._generated_config_files(): + if not spec["path"].exists(): + log_to_stderr(f"Creating BBOT {spec['label']} at {spec['path']}") + self._write_config_template(spec["path"], spec["content"], secret=spec["secret"]) + + def reset_config_files(self, labels): + """Regenerate the named generated config files (`"config"` and/or + `"secrets"`) from current defaults, backing up existing files to + `*.bak`. Returns the backup paths. + + Destructive: a regenerated file is a fresh commented template, so any + options the user uncommented in it are not carried over. + """ + mkdir(self.core.files_config.config_dir) + backups = [] + for spec in self._generated_config_files(): + if spec["label"] not in labels: + continue + path = spec["path"] + if path.exists(): + backup = self._next_backup_path(path) + # copy2 preserves the file's permissions, so a backup of a + # hardened secrets.yml stays just as locked-down + shutil.copy2(path, backup) + backups.append(backup) + self._write_config_template(path, spec["content"], secret=spec["secret"]) + return backups - # ensure bbot.yml - if not files.config_filename.exists(): - log_to_stderr(f"Creating BBOT config at {files.config_filename}") - no_secrets_config = self.core.no_secrets_config(config_obj) - yaml_str = yaml.dump(no_secrets_config, sort_keys=False) - yaml_str = comment_notice + "\n".join(f"# {line}" for line in yaml_str.splitlines()) - with open(str(files.config_filename), "w") as f: + @staticmethod + def _next_backup_path(path): + """First free `.bak`, `.bak.1`, `.bak.2`, ... so an + existing backup from a previous reset isn't clobbered.""" + backup = path.with_name(path.name + ".bak") + i = 1 + while backup.exists(): + backup = path.with_name(f"{path.name}.bak.{i}") + i += 1 + return backup + + @classmethod + def _write_config_template(cls, path, config_dict, secret=False): + header = ( + "# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\n" + "# Please be sure to uncomment when inserting API keys, etc.\n" + ) + yaml_str = yaml.dump(config_dict, sort_keys=False) + yaml_str = header + "\n".join(f"# {line}" for line in yaml_str.splitlines()) + if secret: + cls._write_secret_text(path, yaml_str) + else: + with open(str(path), "w") as f: f.write(yaml_str) - # ensure secrets.yml - if not files.secrets_filename.exists(): - log_to_stderr(f"Creating BBOT secrets at {files.secrets_filename}") - secrets_only_config = self.core.secrets_only_config(config_obj) - yaml_str = yaml.dump(secrets_only_config, sort_keys=False) - yaml_str = comment_notice + "\n".join(f"# {line}" for line in yaml_str.splitlines()) - with open(str(files.secrets_filename), "w") as f: - f.write(yaml_str) - files.secrets_filename.chmod(0o600) + @staticmethod + def _write_secret_text(path, text): + """Write a secrets file so it is never readable by anyone but the owner, + even for an instant. The content is written to a private temp file + (mkstemp creates it 0600) and atomically renamed into place; an existing + file's own permissions are preserved in case the user hardened them + further (e.g. 0400). If owner-only permissions can't be guaranteed, the + secret is not written.""" + path = Path(path) + mode = 0o600 + with suppress(FileNotFoundError): + existing_mode = stat.S_IMODE(path.stat().st_mode) + # keep the user's perms only if they're already owner-only + if not existing_mode & 0o077: + mode = existing_mode + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp") + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w") as f: + f.write(text) + if stat.S_IMODE(os.stat(tmp).st_mode) & 0o077: + raise BBOTError(f"Refusing to write secrets to {path}: could not restrict permissions to owner-only") + os.replace(tmp, str(path)) + except BaseException: + with suppress(FileNotFoundError): + os.unlink(tmp) + raise MODULE_LOADER = ModuleLoader() diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 450282a479..b5a242e6bb 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -411,6 +411,16 @@ def create_parser(self, *args, **kwargs): misc = p.add_argument_group(title="Misc") misc.add_argument("--version", action="store_true", help="show BBOT version and exit") + misc.add_argument( + "--reset-config", + action="store_true", + help="Regenerate bbot.yml from current defaults (overwrites; backs up to .bak)", + ) + misc.add_argument( + "--reset-secrets", + action="store_true", + help="Regenerate secrets.yml from current defaults (overwrites; backs up to .bak)", + ) misc.add_argument("--proxy", help="Use this proxy for all HTTP requests", metavar="HTTP_PROXY") misc.add_argument( "--no-proxy", diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 9979083a1f..c828aa2a92 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -1,3 +1,4 @@ +import stat import yaml from ..bbot_fixtures import * @@ -919,3 +920,99 @@ async def test_cli_no_color(monkeypatch): preset = Preset() preset.parse_args() assert os.environ.get("NO_COLOR") == "1" + + +@pytest.mark.asyncio +async def test_cli_reset_config(monkeypatch, caplog, tmp_path): + from bbot.core import CORE + + monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) + monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + caplog.set_level(logging.INFO) + + files = CORE.files_config + monkeypatch.setattr(files, "config_dir", tmp_path) + monkeypatch.setattr(files, "config_filename", tmp_path / "bbot.yml") + monkeypatch.setattr(files, "secrets_filename", tmp_path / "secrets.yml") + config_file = tmp_path / "bbot.yml" + secrets_file = tmp_path / "secrets.yml" + + # a customized bbot.yml on disk + config_file.write_text("scope:\n strict: false\n") + + # without --yes and no TTY, it refuses and changes nothing + monkeypatch.setattr("sys.argv", ["bbot", "--reset-config"]) + caplog.clear() + await cli._main() + assert "Refusing to reset config without confirmation" in caplog.text + assert not (tmp_path / "bbot.yml.bak").exists() + assert config_file.read_text() == "scope:\n strict: false\n" + + # --reset-config -y regenerates bbot.yml and backs it up, but never touches secrets.yml + monkeypatch.setattr("sys.argv", ["bbot", "--reset-config", "-y"]) + caplog.clear() + await cli._main() + assert "Regenerated config files" in caplog.text + assert (tmp_path / "bbot.yml.bak").is_file() + assert (tmp_path / "bbot.yml.bak").read_text() == "scope:\n strict: false\n" + assert "# NOTICE" in config_file.read_text() + assert not (tmp_path / "secrets.yml.bak").exists() + + # --reset-secrets -y regenerates secrets.yml (owner-only) and backs it up + monkeypatch.setattr("sys.argv", ["bbot", "--reset-secrets", "-y"]) + caplog.clear() + await cli._main() + assert (tmp_path / "secrets.yml.bak").is_file() + assert stat.S_IMODE(secrets_file.stat().st_mode) & 0o077 == 0 + + +@pytest.mark.asyncio +async def test_cli_reset_config_hint(monkeypatch, caplog, tmp_path): + from bbot.core import CORE + + monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) + monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + caplog.set_level(logging.INFO) + + files = CORE.files_config + monkeypatch.setattr(files, "config_dir", tmp_path) + monkeypatch.setattr(files, "config_filename", tmp_path / "bbot.yml") + monkeypatch.setattr(files, "secrets_filename", tmp_path / "secrets.yml") + + # a bbot.yml carrying an option that no longer exists (e.g. left over from + # an older version of BBOT), loaded into the config + (tmp_path / "bbot.yml").write_text("scope:\n strct: false\n") + monkeypatch.setattr(CORE, "_custom_config", {"scope": {"strct": False}}) + + monkeypatch.setattr("sys.argv", ["bbot", "-t", "example.com"]) + caplog.clear() + await cli._main() + + # validation fails on the bad option, and (because it lives in bbot.yml) we + # point the user at --reset-config -- but not at --reset-secrets + assert "scope.strct" in caplog.text + assert "regenerating from current defaults with: bbot --reset-config" in caplog.text + assert "--reset-secrets" not in caplog.text + + +@pytest.mark.asyncio +async def test_cli_reset_config_hint_skips_cli_typo(monkeypatch, caplog, tmp_path): + from bbot.core import CORE + + monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) + monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) + caplog.set_level(logging.INFO) + + files = CORE.files_config + monkeypatch.setattr(files, "config_dir", tmp_path) + monkeypatch.setattr(files, "config_filename", tmp_path / "bbot.yml") + monkeypatch.setattr(files, "secrets_filename", tmp_path / "secrets.yml") + + # the bad option comes from the CLI, not from any config file on disk + monkeypatch.setattr("sys.argv", ["bbot", "-t", "example.com", "-c", "scope.strct=false"]) + caplog.clear() + await cli._main() + + # validation still fails, but there's nothing to reset -- so no hint + assert "scope.strct" in caplog.text + assert "regenerating from current defaults" not in caplog.text diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 4c4d0c0377..fa2be51253 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -1,3 +1,6 @@ +import stat +import tempfile + from ..bbot_fixtures import * # noqa F401 from bbot.scanner import Scanner, Preset @@ -1446,3 +1449,117 @@ def test_all_presets_ignores_non_preset_yaml(tmp_path): preset_mod.DEFAULT_PRESETS = orig_default_presets preset_mod.PRESET_PATH = orig_preset_path path_mod.PRESET_PATH = orig_path_singleton + + +def test_config_isolated_during_tests(): + # the suite must never resolve to the user's real ~/.config/bbot + from bbot.core import CORE + + config_dir = CORE.files_config.config_dir + real_config_dir = (Path.home() / ".config" / "bbot").resolve() + assert config_dir != real_config_dir + assert str(config_dir).startswith(str(Path(tempfile.gettempdir()).resolve())) + + +def test_config_reset(tmp_path, monkeypatch): + from bbot.core import CORE + from bbot.core.modules import MODULE_LOADER + + files = CORE.files_config + monkeypatch.setattr(files, "config_dir", tmp_path) + monkeypatch.setattr(files, "config_filename", tmp_path / "bbot.yml") + monkeypatch.setattr(files, "secrets_filename", tmp_path / "secrets.yml") + config_file = tmp_path / "bbot.yml" + secrets_file = tmp_path / "secrets.yml" + + # first run: files don't exist -> generated as commented templates + MODULE_LOADER.ensure_config_files() + assert config_file.is_file() and secrets_file.is_file() + # secrets.yml is owner-only from the start + assert stat.S_IMODE(secrets_file.stat().st_mode) == 0o600 + + # resetting "config" backs up the existing file and must not touch + # secrets.yml (where API keys live) + config_file.write_text("scope:\n strict: false\n") + secrets_before = secrets_file.read_text() + backups = MODULE_LOADER.reset_config_files(["config"]) + assert set(backups) == {tmp_path / "bbot.yml.bak"} + assert (tmp_path / "bbot.yml.bak").read_text() == "scope:\n strict: false\n" + assert secrets_file.read_text() == secrets_before + # the regenerated file is a fresh commented template + assert "# NOTICE" in config_file.read_text() + + # a backup of a hardened secrets.yml keeps its tightened permissions + secrets_file.chmod(0o400) + backups = MODULE_LOADER.reset_config_files(["secrets"]) + assert set(backups) == {tmp_path / "secrets.yml.bak"} + assert stat.S_IMODE((tmp_path / "secrets.yml.bak").stat().st_mode) == 0o400 + # the regenerated secrets.yml is still owner-only + assert stat.S_IMODE(secrets_file.stat().st_mode) & 0o077 == 0 + + # a second reset must not clobber the first backup + backups2 = MODULE_LOADER.reset_config_files(["secrets"]) + assert set(backups2) == {tmp_path / "secrets.yml.bak.1"} + assert (tmp_path / "secrets.yml.bak").is_file() + + +def test_config_reset_both(tmp_path, monkeypatch): + from bbot.core import CORE + from bbot.core.modules import MODULE_LOADER + + files = CORE.files_config + monkeypatch.setattr(files, "config_dir", tmp_path) + monkeypatch.setattr(files, "config_filename", tmp_path / "bbot.yml") + monkeypatch.setattr(files, "secrets_filename", tmp_path / "secrets.yml") + config_file = tmp_path / "bbot.yml" + + MODULE_LOADER.ensure_config_files() + + # reset both at once -> both backed up + backups = MODULE_LOADER.reset_config_files(["config", "secrets"]) + assert {b.name for b in backups} == {"bbot.yml.bak", "secrets.yml.bak"} + + # resetting only "secrets" leaves bbot.yml untouched + config_before = config_file.read_text() + backups = MODULE_LOADER.reset_config_files(["secrets"]) + assert set(backups) == {tmp_path / "secrets.yml.bak.1"} + assert config_file.read_text() == config_before + + +def test_config_secret_file_permissions(tmp_path): + from bbot.core.modules import MODULE_LOADER + + target = tmp_path / "secrets.yml" + + # a brand-new secret file is owner-only, never world/group readable + MODULE_LOADER._write_secret_text(target, "secret a") + assert stat.S_IMODE(target.stat().st_mode) == 0o600 + assert target.read_text() == "secret a" + + # the user hardened it further -> rewrite preserves the tighter perms + target.chmod(0o400) + MODULE_LOADER._write_secret_text(target, "secret b") + assert stat.S_IMODE(target.stat().st_mode) == 0o400 + assert target.read_text() == "secret b" + + # an existing file with loose perms -> tightened back to owner-only + target.chmod(0o644) + MODULE_LOADER._write_secret_text(target, "secret c") + assert stat.S_IMODE(target.stat().st_mode) == 0o600 + assert target.read_text() == "secret c" + + +def test_config_secret_file_refuses_insecure(tmp_path, monkeypatch): + from bbot.core.modules import MODULE_LOADER + from bbot.errors import BBOTError + + target = tmp_path / "secrets.yml" + + # simulate a filesystem where we can't restrict permissions: the secret is + # not written, and no temp file is left behind + real_fchmod = os.fchmod + monkeypatch.setattr(os, "fchmod", lambda fd, mode: real_fchmod(fd, 0o644)) + with pytest.raises(BBOTError, match="could not restrict permissions"): + MODULE_LOADER._write_secret_text(target, "secret stuff") + assert not target.exists() + assert list(tmp_path.glob(".secrets.yml.*")) == []