fix(engine): resolve memory and goroutine leaks in embedded engine usage (#7503)

.github/workflows/compat-checks.yaml

@@ -13,7 +13,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go/compat-checks@v1
         with:
           go-version: "stable"

.github/workflows/flamegraph.yaml

@@ -12,7 +12,7 @@ jobs:
       TARGET_URL: "http://honey.scanme.sh/-/?foo=bar"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - run: make build

.github/workflows/fuzz.yaml

@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.discover.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
         with:
           go-version-file: go.tool.mod
@@ -37,7 +37,7 @@ jobs:
       GOFUZZ_PACKAGE: ${{ matrix.pkg }}
       FUZZ_DURATION: 15m
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
         with:
           go-version-file: go.tool.mod

.github/workflows/generate-docs.yaml

@@ -11,7 +11,7 @@ jobs:
     if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/actions/setup/git@v1
       - run: make syntax-docs

.github/workflows/generate-pgo.yaml

@@ -18,7 +18,7 @@ jobs:
       TARGET_LIST: "/tmp/targets.txt"
       PROFILE_MEM: "/tmp/nuclei-profile"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - run: |

.github/workflows/govulncheck.yaml

@@ -16,7 +16,7 @@ jobs:
     env:
       OUTPUT: "/tmp/results.sarif"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
         with:
           go-version-file: go.tool.mod

.github/workflows/memogen.yaml

@@ -14,7 +14,7 @@ jobs:
     if: ${{ github.repository == 'projectdiscovery/nuclei' && !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/actions/setup/git@v1
       - run: make memogen

.github/workflows/perf-regression.yaml

@@ -11,7 +11,7 @@ jobs:
     env:
       BENCH_OUT: "/tmp/bench.out"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - run: make build-test

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
     needs: ["pgo"]
     runs-on: ubuntu-latest-16-cores
     steps: 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with: 
           fetch-depth: 0
       - uses: actions/download-artifact@v8

.github/workflows/tests.yaml

@@ -22,9 +22,14 @@ jobs:
     if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/nuclei-action/cache@v3
       - uses: projectdiscovery/actions/golangci-lint/v2@v1
+      # go vet runs once here (not per-OS in the tests matrix): it is platform
+      # independent and folding it into the lint gate fails fast on a cheap
+      # runner before the heavy test/integration jobs start.
+      - run: make vet
 
   tests:
     name: "Tests"
@@ -37,7 +42,7 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     runs-on: "${{ matrix.os }}"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - uses: projectdiscovery/actions/free-disk-space@v1
@@ -49,14 +54,34 @@ jobs:
           misc-packages: 'false'
           docker-images: 'false'
           tools-cache: 'false'
-      - run: make vet
       - run: make build
-      - run: make test
+      # The data-race detector is OS-independent, so we only pay its ~2-3x cost
+      # on ubuntu; windows/macOS run the same suite without instrumentation.
+      - name: "Unit tests (race)"
+        run: make test
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        env:
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
+      - name: "Unit tests"
+        run: make test RACE=
+        if: ${{ matrix.os != 'ubuntu-latest' }}
         env:
           PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
       - run: go run -race . -l ${{ github.workspace }}/internal/tests/functional/testdata/targets.txt -id tech-detect,tls-version
         if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099
         working-directory: cmd/nuclei/
+      # Hermetic HTTP-engine scale regression: stands up many loopback hosts and
+      # asserts finding parity across a diverse template set (per-host pool,
+      # connection-reuse and http->https tracker + cookie-reuse). Race
+      # build runs on ubuntu only (same rationale as the unit tests above);
+      # other OSes run it plain. Large runs stay opt-in locally via
+      # NUCLEI_SCALE_HOSTS.
+      - name: "Scale regression (race)"
+        run: go test -tags=regression -race -timeout 15m ./lib/tests -run TestScaleRegression
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+      - name: "Scale regression"
+        run: make regression
+        if: ${{ matrix.os != 'ubuntu-latest' }}
 
   sdk:
     name: "Run example SDK"
@@ -65,7 +90,7 @@ jobs:
     env:
       GITHUB_TOKEN: "${{ github.token }}"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - name: "Simple"
@@ -90,9 +115,10 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/python@v1
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/nuclei-action/cache@v3
       - run: make integration
         env:
           PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
@@ -109,9 +135,10 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/python@v1
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/nuclei-action/cache@v3
       - uses: projectdiscovery/nuclei-action@v3
         with:
           version: latest
@@ -125,7 +152,7 @@ jobs:
     env:
       GITHUB_TOKEN: "${{ github.token }}"
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/nuclei-action/cache@v3
       - run: make template-validate
@@ -139,19 +166,45 @@ jobs:
       contents: read
       security-events: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: github/codeql-action/init@v4
         with:
           languages: 'go'
       - uses: github/codeql-action/autobuild@v4
       - uses: github/codeql-action/analyze@v4
+        with:
+          output: sarif-results
+          upload: never
+          category: "/language:go"
+      # internal/fuzzplayground is a deliberately vulnerable mock server used to
+      # exercise nuclei's fuzzing templates (CMDI/SQLi/SSRF/etc. are by design),
+      # so its CodeQL alerts are expected false positives. paths-ignore has no
+      # effect for compiled languages, so drop those results from the SARIF
+      # before upload instead.
+      - name: "Filter fuzzplayground alerts from SARIF"
+        run: |
+          for sarif in sarif-results/*.sarif; do
+            [ -e "$sarif" ] || continue
+            jq '
+              .runs[].results |= map(
+                select(
+                  [ .locations[]?.physicalLocation.artifactLocation.uri // "" ]
+                  | any(startswith("internal/fuzzplayground/")) | not
+                )
+              )
+            ' "$sarif" > "$sarif.filtered" && mv "$sarif.filtered" "$sarif"
+          done
+      - uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: sarif-results
+          category: "/language:go"
 
   release:
     name: "Release test"
     needs: ["tests"]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: projectdiscovery/actions/setup/go@v1
         with:
           go-version: "stable"

.github/workflows/typos.yaml

@@ -16,5 +16,5 @@ jobs:
     if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: crate-ci/typos@v1.47.2

Makefile

@@ -23,7 +23,7 @@ endif
 
 .PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs fuzz fuzz-ci fuzz-tools
 .PHONY: devtools-tsgen docs docgen dsl-docs functional go-build lint lint-strict fuzzplayground syntax-docs
-.PHONY: integration integration-debug jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint
+.PHONY: integration integration-debug regression jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint
 .PHONY: tidy ts verify download vet template-validate build-fuzz discover-fuzz-packages
 
 all: build
@@ -82,7 +82,11 @@ syntax-docs: docgen
 syntax-docs:
 	./bin/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json
 
-test: GOFLAGS = -race -v -timeout 1h -count 1
+# RACE controls the data-race detector (on by default for local runs). CI builds
+# the race variant on a single OS and passes RACE= elsewhere, since data races are
+# OS-independent and the detector costs ~2-3x build+run time on every runner.
+RACE ?= -race
+test: GOFLAGS = $(RACE) -v -timeout 1h -count 1
 test:
 	$(GOTEST) $(GOFLAGS) ./...
 
@@ -92,6 +96,12 @@ integration:
 integration-debug:
 	$(GOTEST) -tags=integration ./internal/tests/integration -v $(GO_TEST_ARGS) -args $(INTEGRATION_ARGS)
 
+# Opt-in HTTP engine scale regression harness (not part of CI). Stands up many
+# loopback hosts and asserts finding parity across a diverse template set.
+# Override host count with NUCLEI_SCALE_HOSTS, e.g. NUCLEI_SCALE_HOSTS=500 make regression
+regression:
+	$(GOTEST) -tags=regression -timeout 30m ./lib/tests -run TestScaleRegression -v
+
 functional: build
 	@release_binary="$$(command -v nuclei.exe 2>/dev/null || command -v nuclei 2>/dev/null)"; \
 	if [ -z "$$release_binary" ]; then \

cmd/nuclei/main.go

@@ -15,12 +15,12 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	_pdcp "github.com/projectdiscovery/nuclei/v3/internal/pdcp"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"github.com/rs/xid"
-	"gopkg.in/yaml.v2"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger/levels"
@@ -418,6 +418,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
 		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
+		flagSet.BoolVar(&options.PerHostRateLimit, "per-host-rate-limit", false, "enable per-host rate limiting (global rate limit becomes unlimited when enabled)"),
 		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
@@ -446,6 +447,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		}),
 		flagSet.DurationVarP(&options.InputReadTimeout, "input-read-timeout", "irt", time.Duration(3*time.Minute), "timeout on input read"),
 		flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
+		flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
 		flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
 	)