Fix sporadic nuclei test failures from stale template state / Nuclei Environment isolation

bbot/modules/nuclei.py

@@ -1,6 +1,10 @@
+import asyncio
 import json
+import os
+import shutil
 import yaml
 from itertools import islice
+
 from bbot.modules.base import BaseModule
 
 
@@ -62,21 +66,27 @@ class nuclei(BaseModule):
     _batch_size = 200
 
     async def setup(self):
-        # attempt to update nuclei templates
-        self.nuclei_templates_dir = self.helpers.tools_dir / "nuclei-templates"
-        self.info("Updating Nuclei templates")
-        update_results = await self.run_process(
-            ["nuclei", "-update-template-dir", self.nuclei_templates_dir, "-update-templates"]
-        )
-        if update_results.stderr:
-            if "Successfully downloaded nuclei-templates" in update_results.stderr:
-                self.success("Successfully updated nuclei templates")
-            elif "No new updates found for nuclei templates" in update_results.stderr:
-                self.info("Nuclei templates already up-to-date")
-            else:
-                self.warning(f"Failure while updating nuclei templates: {update_results.stderr}")
-        else:
-            self.warning("Error running nuclei template update command")
+        # All nuclei state lives under one bbot-owned dir so we can wipe it on
+        # corruption without touching the user's own ~/.config/nuclei. See
+        # _nuclei_env() for how the subprocess env pins config/cache here.
+        self.nuclei_state_dir = self.helpers.tools_dir / "nuclei-state"
+        self.nuclei_config_dir = self.nuclei_state_dir / "config"
+        self.nuclei_cache_dir = self.nuclei_state_dir / "cache"
+        self.nuclei_templates_dir = self.nuclei_state_dir / "templates"
+        self.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
+        self.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
+        await self._update_templates()
+        # nuclei writes its version marker before the tarball finishes extracting,
+        # so a killed update can leave the marker pointing at an empty dir and
+        # every subsequent run reports "up-to-date." Verify and repair once.
+        if not self._templates_installed():
+            self.warning("Nuclei templates appear incomplete; wiping isolated state and re-downloading")
+            shutil.rmtree(self.nuclei_state_dir, ignore_errors=True)
+            self.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
+            self.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
+            await self._update_templates()
+            if not self._templates_installed():
+                return False, "Failed to install nuclei templates after retry"
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.mode = self.config.get("mode", "severe").lower()
         self.ratelimit = int(self.config.get("ratelimit", 150))
@@ -257,7 +267,9 @@ async def execute_nuclei(self, nuclei_input):
         stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status)
         try:
             with open(stats_file, "w") as stats_fh:
-                async for line in self.run_process_live(command, input=nuclei_input, stderr=stats_fh):
+                async for line in self.run_process_live(
+                    command, input=nuclei_input, stderr=stats_fh, env=self._nuclei_env()
+                ):
                     try:
                         j = json.loads(line)
                     except json.decoder.JSONDecodeError:
@@ -317,6 +329,72 @@ async def cleanup(self):
         resume_file = self.helpers.current_dir / "resume.cfg"
         resume_file.unlink(missing_ok=True)
 
+    def _nuclei_env(self):
+        # Allowlist env vars: nuclei reads PDCP_API_KEY, GITHUB_TOKEN, AWS_*,
+        # AZURE_*, etc. directly, so os.environ.copy() would silently change
+        # its behavior (e.g. upload findings to ProjectDiscovery Cloud under
+        # the user's account). XDG_{CONFIG,CACHE}_HOME pin nuclei's config and
+        # cache under nuclei-state/ without inheriting the user's HOME.
+        keep = {
+            "PATH",
+            "LD_LIBRARY_PATH",
+            "LANG",
+            "LC_ALL",
+            "TZ",
+            "HTTP_PROXY",
+            "HTTPS_PROXY",
+            "NO_PROXY",
+            "http_proxy",
+            "https_proxy",
+            "no_proxy",
+        }
+        env = {k: v for k, v in os.environ.items() if k in keep}
+        env["XDG_CONFIG_HOME"] = str(self.nuclei_config_dir)
+        env["XDG_CACHE_HOME"] = str(self.nuclei_cache_dir)
+        return env
+
+    async def _update_templates(self):
+        self.info("Updating Nuclei templates")
+        # shield so an outer cancel can't kill the subprocess mid-extract and
+        # corrupt the templates dir
+        update_results = await asyncio.shield(
+            self.run_process(
+                ["nuclei", "-update-template-dir", self.nuclei_templates_dir, "-update-templates"],
+                env=self._nuclei_env(),
+            )
+        )
+        outcome = self._classify_update_stderr(update_results.stderr)
+        if outcome == "updated":
+            self.success("Successfully updated nuclei templates")
+        elif outcome == "up-to-date":
+            self.info("Nuclei templates already up-to-date")
+        else:
+            self.warning(f"Failure while updating nuclei templates: {update_results.stderr or '<no stderr>'}")
+
+    # nuclei's success messaging has drifted across releases (installed / updated /
+    # downloaded). Match any of them so a future rename doesn't silently downgrade
+    # a clean install to a "Failure while updating" warning.
+    _UPDATE_SUCCESS_MARKERS = (
+        "Successfully installed nuclei-templates",
+        "Successfully updated nuclei-templates",
+        "Successfully downloaded nuclei-templates",
+    )
+    _UPDATE_NOOP_MARKER = "No new updates found for nuclei templates"
+
+    @classmethod
+    def _classify_update_stderr(cls, stderr):
+        if not stderr:
+            return "failure"
+        if any(m in stderr for m in cls._UPDATE_SUCCESS_MARKERS):
+            return "updated"
+        if cls._UPDATE_NOOP_MARKER in stderr:
+            return "up-to-date"
+        return "failure"
+
+    def _templates_installed(self):
+        http_dir = self.nuclei_templates_dir / "http"
+        return http_dir.is_dir() and any(http_dir.iterdir())
+
     async def filter_event(self, event):
         if self.config.get("directory_only", True):
             if "endpoint" in event.tags:

bbot/test/test_step_2/module_tests/test_module_nuclei.py

@@ -15,7 +15,7 @@ class TestNucleiManual(ModuleTestBase):
                 "mode": "manual",
                 "concurrency": 2,
                 "ratelimit": 10,
-                "templates": "/tmp/.bbot_test/tools/nuclei-templates/http/miscellaneous/",
+                "templates": "/tmp/.bbot_test/tools/nuclei-state/templates/http/miscellaneous/",
                 "interactsh_disable": True,
                 "directory_only": False,
             }
@@ -69,7 +69,7 @@ class TestNucleiSevere(TestNucleiManual):
             "nuclei": {
                 "mode": "severe",
                 "concurrency": 1,
-                "templates": "/tmp/.bbot_test/tools/nuclei-templates/http/vulnerabilities/generic/generic-env.yaml",
+                "templates": "/tmp/.bbot_test/tools/nuclei-state/templates/http/vulnerabilities/generic/generic-env.yaml",
             }
         },
         "interactsh_disable": True,
@@ -114,7 +114,7 @@ class TestNucleiBudget(TestNucleiManual):
                 "mode": "budget",
                 "concurrency": 1,
                 "tags": "spiderfoot",
-                "templates": "/tmp/.bbot_test/tools/nuclei-templates/exposed-panels/spiderfoot.yaml",
+                "templates": "/tmp/.bbot_test/tools/nuclei-state/templates/exposed-panels/spiderfoot.yaml",
                 "interactsh_disable": True,
             }
         }
@@ -156,6 +156,82 @@ def check(self, module_test, events):
         assert "-retries 1" in open(module_test.scan.home / "debug.log").read()
 
 
+class TestNucleiEnvIsolation(TestNucleiManual):
+    """Set hostile env vars before the scan and verify _nuclei_env() filters them out.
+
+    The base TestNucleiManual assertions double as a regression check: nuclei must
+    still find and run templates even when the user's env has DISABLE_*_DOWNLOAD,
+    XDG_CONFIG_HOME, etc. set.
+    """
+
+    leaky_env = {
+        "PDCP_API_KEY": "user-pdcp-key-must-not-leak",
+        "PDCP_TEAM_ID": "user-team",
+        "XDG_CONFIG_HOME": "/tmp/.bbot_test/xdg-leak-config",
+        "XDG_CACHE_HOME": "/tmp/.bbot_test/xdg-leak-cache",
+        "GITHUB_TOKEN": "ghp_usertoken",
+        "GITHUB_TEMPLATE_REPO": "attacker/private-templates",
+        "GITLAB_TOKEN": "glpat-usertoken",
+        "AWS_ACCESS_KEY": "AKIA-user",
+        "AWS_SECRET_KEY": "user-aws-secret",
+        "AZURE_CLIENT_SECRET": "azure-secret",
+        "NUCLEI_SIGNATURE_PUBLIC_KEY": "user-pubkey",
+        "DISABLE_NUCLEI_TEMPLATES_PUBLIC_DOWNLOAD": "true",
+    }
+
+    async def setup_before_prep(self, module_test):
+        await super().setup_before_prep(module_test)
+        for k, v in self.leaky_env.items():
+            module_test.monkeypatch.setenv(k, v)
+
+    # XDG_{CONFIG,CACHE}_HOME are set by _nuclei_env() itself to our paths,
+    # so they appear in env but must override (not echo back) the user's values.
+    _xdg_overridden = {"XDG_CONFIG_HOME", "XDG_CACHE_HOME"}
+
+    def check(self, module_test, events):
+        super().check(module_test, events)
+        nuclei = module_test.scan.modules["nuclei"]
+        env = nuclei._nuclei_env()
+        for k, v in self.leaky_env.items():
+            if k in self._xdg_overridden:
+                assert env.get(k) != v, f"{k} leaked user value into nuclei env"
+            else:
+                assert k not in env, f"{k} leaked into nuclei subprocess env"
+        assert env["XDG_CONFIG_HOME"] == str(nuclei.nuclei_config_dir)
+        assert env["XDG_CACHE_HOME"] == str(nuclei.nuclei_cache_dir)
+        # HOME is intentionally NOT forwarded — nuclei must rely on the XDG
+        # vars we set, not on the user's home dir.
+        assert "HOME" not in env
+
+
+def test_nuclei_classify_update_stderr():
+    """Regression: nuclei's success line has changed wording across releases
+    ("downloaded" → "installed" → "updated"). All three must classify as
+    success so a fresh install isn't mis-logged as a failure.
+    """
+    from bbot.modules.nuclei import nuclei
+
+    c = nuclei._classify_update_stderr
+    # First-time install (what nuclei v3.8 prints on a clean state dir).
+    assert (
+        c(
+            "[INF] nuclei-templates are not installed, installing...\n[INF] Successfully installed nuclei-templates at /tmp/x"
+        )
+        == "updated"
+    )
+    # Version bump (newer phrasing).
+    assert c("[INF] Successfully updated nuclei-templates (v10.4.3) to /tmp/x. GoodLuck!") == "updated"
+    # Legacy phrasing — kept for older nuclei builds.
+    assert c("[INF] Successfully downloaded nuclei-templates") == "updated"
+    # Already up-to-date.
+    assert c("[INF] No new updates found for nuclei templates") == "up-to-date"
+    # Benign first-run noise must NOT trick us into reporting success.
+    assert c("[ERR] Could not copy nuclei ignore file ...: source file doesn't exist") == "failure"
+    # Empty / None stderr → failure (process produced nothing).
+    assert c("") == "failure"
+    assert c(None) == "failure"
+
+
 class TestNucleiCustomHeaders(TestNucleiManual):
     custom_headers = {"testheader1": "test1", "testheader2": "test2"}
     config_overrides = TestNucleiManual.config_overrides