From 441b37db748bb2cf7d7875a308af9bf9f5691b6f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 27 Jul 2026 23:54:43 +0400 Subject: [PATCH 1/7] idle conns --- lib/config.go | 8 +++ .../http/httpclientpool/clientpool.go | 54 ++++++++++++++++--- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/lib/config.go b/lib/config.go index 66a2a9ef4c..fb471aba00 100644 --- a/lib/config.go +++ b/lib/config.go @@ -754,3 +754,11 @@ func WithTemporaryDirectory(parentDir string) NucleiSDKOptions { return nil } } + +// WithPerHostRateLimit applies the rate limit per host instead of globally. +func WithPerHostRateLimit(enabled bool) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.PerHostRateLimit = enabled + return nil + } +} diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 9606ba022f..aa07c3a923 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -322,13 +322,24 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin retryableHttpOptions.Timeout = configuration.ResponseHeaderTimeout } - maxIdleConns := 4 - maxIdleConnsPerHost := 4 + // Size the idle pool to the number of requests this process can have in + // flight against a SINGLE host, so a completed request hands its connection + // to the next one instead of the transport closing it. + // + // In template-spray, per-host in-flight is TemplateThreads: each of the N + // templates running in parallel issues one request per host (BulkSize is + // hosts-in-parallel, which does not add per-host concurrency). A template + // with its own `threads:`, or payload/fuzz concurrency, can exceed that. + // + // A fixed 4 (the previous default) throttles reuse badly once per-host + // concurrency rises: measured on 10 hosts at TemplateThreads=20, idle=4 gave + // 43% reuse / 572 rps versus 81% / 1088 rps at idle=16 — a 1.9x throughput + // difference from this constant alone. The curve is flat past + // idle ~= per-host in-flight, so overshooting is cheap and undershooting is + // not; the cap only exists to bound retained sockets (hosts * idle fds). + maxIdleConnsPerHost := perHostIdleConns(options, configuration) + maxIdleConns := maxIdleConnsPerHost maxConnsPerHost := 0 // unlimited by default; the SPM handler controls concurrency - if configuration.Threads > 0 { - maxIdleConnsPerHost = configuration.Threads - maxIdleConns = configuration.Threads - } disableKeepAlives := configuration.Connection != nil && configuration.Connection.DisableKeepAlive @@ -705,3 +716,34 @@ func RecordHTTPToHTTPSPortMismatch(options *types.Options, hostname string) { tracker.RecordHTTPToHTTPSPort(hostname) } + +// minIdleConnsPerHost / maxIdleConnsPerHostCap bound the derived idle pool. +// The floor keeps behaviour sane when concurrency options are unset; the ceiling +// bounds retained file descriptors, since a process holds up to +// hosts * maxIdleConnsPerHost idle sockets. +const ( + minIdleConnsPerHost = 4 + maxIdleConnsPerHostCap = 64 +) + +// perHostIdleConns derives the idle-connection budget for one host's transport +// from the concurrency that can actually target that host. +func perHostIdleConns(options *types.Options, configuration *Configuration) int { + n := minIdleConnsPerHost + // Templates running in parallel each issue one request per host. + if options != nil && options.TemplateThreads > n { + n = options.TemplateThreads + } + // A template declaring its own `threads:` drives that many per host. + if configuration != nil && configuration.Threads > n { + n = configuration.Threads + } + // Payload/fuzz requests within a single template also run concurrently. + if options != nil && options.PayloadConcurrency > n { + n = options.PayloadConcurrency + } + if n > maxIdleConnsPerHostCap { + n = maxIdleConnsPerHostCap + } + return n +} From 40bd7f1777e1ff587a6b46a32acf5675dd172e32 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 30 Jul 2026 16:01:59 +0400 Subject: [PATCH 2/7] strict probe --- README.md | 1 + README_CN.md | 1 + README_ES.md | 1 + README_ID.md | 1 + README_JP.md | 1 + README_KR.md | 1 + README_PT-BR.md | 1 + README_TR.md | 1 + cmd/nuclei/main.go | 1 + internal/runner/reachability.go | 282 +++++++++++++++++++++++++++ internal/runner/reachability_test.go | 108 ++++++++++ internal/runner/runner.go | 24 +++ pkg/input/transform.go | 10 + pkg/input/transform_test.go | 29 +++ pkg/types/types.go | 5 + 15 files changed, 467 insertions(+) create mode 100644 internal/runner/reachability.go create mode 100644 internal/runner/reachability_test.go diff --git a/README.md b/README.md index 28ef44e980..c81b7f7160 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_CN.md b/README_CN.md index a4ce915ca2..502b0102ce 100644 --- a/README_CN.md +++ b/README_CN.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_ES.md b/README_ES.md index 53ef00b81e..dd09e858ea 100644 --- a/README_ES.md +++ b/README_ES.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_ID.md b/README_ID.md index 8d93bb2447..ff494b5b9c 100644 --- a/README_ID.md +++ b/README_ID.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_JP.md b/README_JP.md index 81e8cefe4c..016e20c9ad 100644 --- a/README_JP.md +++ b/README_JP.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_KR.md b/README_KR.md index 85c227cf6a..f5b9d735e1 100644 --- a/README_KR.md +++ b/README_KR.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_PT-BR.md b/README_PT-BR.md index abd3637168..5887dfe18d 100644 --- a/README_PT-BR.md +++ b/README_PT-BR.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/README_TR.md b/README_TR.md index a729de6764..a1feb858e1 100644 --- a/README_TR.md +++ b/README_TR.md @@ -306,6 +306,7 @@ OPTIMIZATIONS: -ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto) -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input + -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index bb8e8dd88a..e794f0614b 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -289,6 +289,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"), + flagSet.BoolVarP(&options.StrictProbe, "strict-probe", "stp", false, "skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"), diff --git a/internal/runner/reachability.go b/internal/runner/reachability.go new file mode 100644 index 0000000000..3575c2c3ec --- /dev/null +++ b/internal/runner/reachability.go @@ -0,0 +1,282 @@ +package runner + +import ( + "context" + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" + sliceutil "github.com/projectdiscovery/utils/slice" +) + +type probeResult int + +const ( + portClosed probeResult = iota // connection refused / host unreachable + portOpen // connect succeeded + portUnknown // timeout / filtered — treat as possibly open +) + +const reachabilityProbeTimeout = 2 * time.Second + +// strictProbeEnabled reports whether the lossless reachability prune +// applies to this run. It is limited to standard target inputs; request-shaped +// (DAST / multi-format) inputs are never pruned, because probing a request +// template as a live service is meaningless and could drop coverage. +func (r *Runner) strictProbeEnabled() bool { + return r.options.StrictProbe && + r.inputProvider != nil && + r.inputProvider.InputType() != provider.MultiFormatInputProvider +} + +// noHTTPServiceReachable reports whether NO target actually speaks HTTP(S). +// +// When it returns true, web templates (HTTP/headless/websocket) can be excluded +// losslessly: a port that does not serve HTTP has no HTTP application, so no +// HTTP template can produce a real finding there. Unlike a bare TCP-open check, +// this catches the open-but-non-HTTP case (e.g. redis on 6379): the port is +// open, but it is not an HTTP service, so HTTP templates are pure waste. +// +// It reuses nuclei's own httpx probe (the same one used for input routing), so +// the determination is consistent with the engine and respects the network +// policy (the probe dials only through the policy-configured httpx client). +// Conservative: an explicit http/https target, any scheme that responds, or the +// inability to probe safely all keep web templates in place. +func (r *Runner) noHTTPServiceReachable() bool { + if r.inputProvider == nil { + return false + } + dialers := protocolstate.GetDialersWithId(r.options.ExecutionId) + if dialers == nil { + return false // cannot probe within policy — keep web templates + } + httpxOptions := httpx.DefaultOptions + if r.options.AliveHttpProxy != "" { + httpxOptions.Proxy = r.options.AliveHttpProxy + } else if r.options.AliveSocksProxy != "" { + httpxOptions.Proxy = r.options.AliveSocksProxy + } + httpxOptions.RetryMax = r.options.Retries + if r.options.Timeout > 0 { + httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second + } + httpxOptions.NetworkPolicy = dialers.NetworkPolicy + client, err := httpx.New(&httpxOptions) + if err != nil { + return false // cannot probe — keep web templates + } + + anyHTTP := false + r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { + if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") { + anyHTTP = true + return false + } + if utils.ProbeURL(mi.Input, client) != "" { + anyHTTP = true + return false + } + return true + }) + return !anyHTTP +} + +// pruneClosedTCPNetworkTemplates removes network(TCP) templates whose declared +// port(s) are definitively closed on every target. Provably lossless: such a +// template cannot connect anywhere, so it cannot produce a finding. +// +// Strictly gated to stay lossless: +// - only applies when EVERY target is a bare host (no explicit port); an +// explicit input port overrides the template port, so we skip those. +// - only single-protocol, network-only templates (a mixed template could have +// a reachable request in another protocol). +// - only TCP: any udp:// address disqualifies the template (a TCP probe says +// nothing about a UDP service — e.g. SNMP/mDNS). +// - only concrete numeric ports; empty/dynamic/service-name ports are kept. +// - prunes only when a port is CLOSED (refused); open or indeterminate keeps it. +func (r *Runner) pruneClosedTCPNetworkTemplates(in []*templates.Template) []*templates.Template { + if r.inputProvider == nil || len(in) == 0 { + return in + } + // Collect hosts; bail out (keep everything) if any input carries a port. + var hosts []string + bareOnly := true + r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { + host, port := hostAndExplicitPort(mi.Input) + if port != "" { + bareOnly = false + return false + } + if host != "" { + hosts = append(hosts, host) + } + return true + }) + if !bareOnly || len(hosts) == 0 { + return in + } + hosts = sliceutil.Dedupe(hosts) + + // Identify prunable candidates and the ports to probe. + candidatePorts := map[int][]string{} + toProbe := map[string]struct{}{} + for i, t := range in { + ports, ok := tcpNetworkOnlyPorts(t) + if !ok { + continue + } + candidatePorts[i] = ports + for _, p := range ports { + toProbe[p] = struct{}{} + } + } + if len(candidatePorts) == 0 { + return in + } + + // A port is "reachable" if open or indeterminate on ANY host. + reachable := map[string]bool{} + for p := range toProbe { + for _, h := range hosts { + res := r.probe(h, p) + if res == portOpen || res == portUnknown { + reachable[p] = true + break + } + } + } + + out := in[:0] + pruned := 0 + for i, t := range in { + if ports, ok := candidatePorts[i]; ok { + anyReachable := false + for _, p := range ports { + if reachable[p] { + anyReachable = true + break + } + } + if !anyReachable { + pruned++ + continue + } + } + out = append(out, t) + } + if pruned > 0 { + gologger.Info().Msgf("reachability prune: excluded %d network(tcp) template[s] targeting only closed ports (lossless)", pruned) + } + return out +} + +// tcpNetworkOnlyPorts returns the concrete numeric TCP ports of a single-protocol +// network template, or ok=false if the template is ineligible for port pruning. +func tcpNetworkOnlyPorts(t *templates.Template) ([]string, bool) { + if len(t.RequestsNetwork)+len(t.RequestsWithTCP) == 0 { + return nil, false + } + // must be network-only (no other protocol request could still be reachable) + otherReqs := len(t.RequestsHTTP) + len(t.RequestsWithHTTP) + len(t.RequestsHeadless) + + len(t.RequestsDNS) + len(t.RequestsFile) + len(t.RequestsSSL) + + len(t.RequestsCode) + len(t.RequestsJavascript) + len(t.Workflows) + if otherReqs > 0 { + return nil, false + } + reqs := make([]*network.Request, 0, len(t.RequestsNetwork)+len(t.RequestsWithTCP)) + reqs = append(reqs, t.RequestsNetwork...) + reqs = append(reqs, t.RequestsWithTCP...) + + var ports []string + for _, req := range reqs { + for _, addr := range req.Address { + if strings.Contains(strings.ToLower(addr), "udp://") { + return nil, false // UDP: a TCP probe cannot prove unreachability + } + } + pp := splitPorts(req.Port) + if len(pp) == 0 { + return nil, false // empty/dynamic port (uses input port) — keep + } + for _, p := range pp { + if !isNumericPort(p) { + return nil, false // service name or template var — keep + } + ports = append(ports, p) + } + } + if len(ports) == 0 { + return nil, false + } + return sliceutil.Dedupe(ports), true +} + +func isNumericPort(p string) bool { + n, err := strconv.Atoi(p) + return err == nil && n > 0 && n < 65536 +} + +// hostAndExplicitPort splits a target into host and explicit port (port empty if +// the target is a bare host). +func hostAndExplicitPort(input string) (string, string) { + if strings.Contains(input, "://") { + if u, err := url.Parse(input); err == nil && u.Hostname() != "" { + return u.Hostname(), u.Port() + } + } + if host, port, err := net.SplitHostPort(input); err == nil { + return host, port + } + return input, "" +} + +// probe classifies reachability of host:port WITHIN the scan's network policy. +// Security first: it never dials a target the policy forbids and never bypasses +// the sandbox — all probing goes through the execution's fastdialer, which +// enforces the same DenyList (‑lna, exclude-targets, private/metadata ranges, +// DNS-rebind protection) as the scan itself. +// +// It reconciles security with losslessness by mapping every uncertain case to +// portUnknown ("keep, don't dial"): a policy-denied host, a missing dialer, or a +// timeout all leave templates in place. Pruning happens only on a definitive, +// policy-permitted, refused connection. +func (r *Runner) probe(host, port string) probeResult { + if !protocolstate.IsHostAllowed(r.options.ExecutionId, host) { + // Policy denies this host: do not dial (security), do not prune (lossless). + return portUnknown + } + dialers := protocolstate.GetDialersWithId(r.options.ExecutionId) + if dialers == nil || dialers.Fastdialer == nil { + // No policy-enforcing dialer available: never fall back to a raw dial. + return portUnknown + } + return classifyDial(dialers.Fastdialer.Dial, net.JoinHostPort(host, port), reachabilityProbeTimeout) +} + +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// classifyDial performs one connect via the supplied (policy-enforcing) dialer. +func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + conn, err := dial(ctx, "tcp", addr) + if err == nil { + _ = conn.Close() + return portOpen + } + if ne, ok := err.(net.Error); ok && ne.Timeout() { + return portUnknown + } + return portClosed +} + diff --git a/internal/runner/reachability_test.go b/internal/runner/reachability_test.go new file mode 100644 index 0000000000..3486a1c08b --- /dev/null +++ b/internal/runner/reachability_test.go @@ -0,0 +1,108 @@ +package runner + +import ( + "net" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + httpproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" + jsproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript" +) + +func TestHostAndExplicitPort(t *testing.T) { + cases := []struct{ in, host, port string }{ + {"127.0.0.1", "127.0.0.1", ""}, + {"127.0.0.1:6379", "127.0.0.1", "6379"}, + {"http://example.com:8080", "example.com", "8080"}, + {"https://example.com", "example.com", ""}, + } + for _, c := range cases { + h, p := hostAndExplicitPort(c.in) + if h != c.host || p != c.port { + t.Errorf("hostAndExplicitPort(%q) = (%q,%q), want (%q,%q)", c.in, h, p, c.host, c.port) + } + } +} + +func TestIsNumericPort(t *testing.T) { + for _, p := range []string{"80", "6379", "65535", "1"} { + if !isNumericPort(p) { + t.Errorf("isNumericPort(%q) = false, want true", p) + } + } + for _, p := range []string{"ftp", "{{Port}}", "0", "70000", "", "80a"} { + if isNumericPort(p) { + t.Errorf("isNumericPort(%q) = true, want false", p) + } + } +} + +// TestTCPNetworkOnlyPorts locks in the losslessness guards: only single-protocol +// network templates with concrete numeric TCP ports are eligible for pruning. +func TestTCPNetworkOnlyPorts(t *testing.T) { + netTmpl := func(addr []string, port string) *templates.Template { + return &templates.Template{RequestsNetwork: []*network.Request{{Address: addr, Port: port}}} + } + + t.Run("concrete tcp port eligible", func(t *testing.T) { + ports, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "6379,6380")) + if !ok || len(ports) != 2 { + t.Fatalf("want ok with 2 ports, got ok=%v ports=%v", ok, ports) + } + }) + t.Run("tls address still tcp", func(t *testing.T) { + if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"tls://{{Hostname}}"}, "6379")); !ok { + t.Fatal("tls:// should remain eligible (still TCP)") + } + }) + t.Run("udp address rejected", func(t *testing.T) { + if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"udp://{{Hostname}}"}, "161")); ok { + t.Fatal("udp:// must be ineligible — a TCP probe cannot prove UDP unreachability") + } + }) + t.Run("service-name port rejected", func(t *testing.T) { + if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "ftp")); ok { + t.Fatal("service-name port must be ineligible") + } + }) + t.Run("dynamic port rejected", func(t *testing.T) { + if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "{{Port}}")); ok { + t.Fatal("template-var port must be ineligible") + } + }) + t.Run("empty port rejected", func(t *testing.T) { + if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "")); ok { + t.Fatal("empty port (uses input port) must be ineligible") + } + }) + t.Run("mixed protocol rejected", func(t *testing.T) { + mixed := netTmpl([]string{"{{Hostname}}"}, "6379") + mixed.RequestsHTTP = []*httpproto.Request{{}} + if _, ok := tcpNetworkOnlyPorts(mixed); ok { + t.Fatal("template with an HTTP request must be ineligible") + } + }) + t.Run("javascript protocol rejected", func(t *testing.T) { + js := &templates.Template{RequestsJavascript: []*jsproto.Request{{}}} + if _, ok := tcpNetworkOnlyPorts(js); ok { + t.Fatal("javascript template must be ineligible (opaque dial protocol)") + } + }) +} + +func TestClassifyDial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Skip("cannot listen:", err) + } + defer ln.Close() + d := &net.Dialer{} + if got := classifyDial(d.DialContext, ln.Addr().String(), reachabilityProbeTimeout); got != portOpen { + t.Errorf("classifyDial(open) = %v, want portOpen", got) + } + // an unused low port on loopback should refuse quickly + if got := classifyDial(d.DialContext, "127.0.0.1:1", reachabilityProbeTimeout); got != portClosed { + t.Errorf("classifyDial(closed) = %v, want portClosed", got) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 98a87e1bf0..d76b24f52c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -64,6 +64,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" + templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" @@ -678,6 +679,15 @@ func (r *Runner) RunEnumeration() error { } executorOpts.WorkflowLoader = workflowLoader + // Lossless reachability prune: if no web port is reachable on any target, + // exclude web-protocol templates at load time — they cannot connect, so + // cannot produce a finding. Reuses the standard protocol-type exclusion. + if r.strictProbeEnabled() && r.noHTTPServiceReachable() { + r.options.ExcludeProtocols = append(r.options.ExcludeProtocols, + templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol, templateTypes.WebsocketProtocol) + gologger.Info().Msgf("reachability prune: no HTTP service on any target; excluding web-protocol templates (lossless)") + } + // If using input-file flags, only load http fuzzing based templates. loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts) if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST { @@ -770,6 +780,14 @@ func (r *Runner) RunEnumeration() error { return errors.Wrap(err, "could not probe http input") } executorOpts.InputHelper.InputsHTTP = inputHelpers + // Under strict-probe, skip web templates per-input for targets that + // probed as non-HTTP (lossless), instead of falling back to raw URL. + executorOpts.InputHelper.StrictProbe = r.strictProbeEnabled() + if r.strictProbeEnabled() { + r.Logger.Info().Msgf("Strict probe enabled: hosts without httpx confirmation are skipped for HTTP/headless templates") + } + } else if r.options.StrictProbe && r.options.DisableHTTPProbe { + r.Logger.Warning().Msgf("strict-probe has no effect with -no-httpx (httpx probing is disabled)") } inputCount := int(r.inputProvider.Count()) @@ -926,6 +944,12 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) return nil, errors.New("no templates provided for scan") } + // Lossless reachability prune (network layer): drop network(tcp) templates + // whose declared ports are definitively closed on every target. + if r.strictProbeEnabled() { + finalTemplates = r.pruneClosedTCPNetworkTemplates(finalTemplates) + } + // pass input provider to engine // TODO: this should be not necessary after r.hmapInputProvider is removed + refactored if r.inputProvider == nil { diff --git a/pkg/input/transform.go b/pkg/input/transform.go index 76f122c2e3..2d1ac46e2f 100644 --- a/pkg/input/transform.go +++ b/pkg/input/transform.go @@ -16,6 +16,10 @@ import ( // Helper is a structure for helping with input transformation type Helper struct { InputsHTTP *hybrid.HybridMap + // StrictProbe, when set, makes web-template input resolution skip inputs that + // were probed and found NOT to be HTTP services (instead of falling back to + // a raw URL). Lossless: a non-HTTP port has no HTTP application to find. + StrictProbe bool } // NewHelper returns a new input helper instance @@ -114,6 +118,12 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo if probed, ok := h.InputsHTTP.Get(input); ok { return string(probed) } + // Strict reachability: this input was probed and is not an HTTP + // service, so skip it for web templates rather than falling back to + // a raw URL. Lossless — a non-HTTP port has no HTTP app to match. + if h.StrictProbe { + return "" + } } // try to parse it as absolute url and return if absUrl, err := urlutil.ParseAbsoluteURL(input, false); err == nil { diff --git a/pkg/input/transform_test.go b/pkg/input/transform_test.go index 4cd866562f..708ded4a9b 100644 --- a/pkg/input/transform_test.go +++ b/pkg/input/transform_test.go @@ -72,3 +72,32 @@ func TestConvertInputToType(t *testing.T) { require.Equal(t, test.result, result, "could not get correct result %+v", test) } } + +func TestStrictProbeSkipsUnconfirmedHTTPInput(t *testing.T) { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + require.NoError(t, err) + t.Cleanup(func() { _ = hm.Close() }) + + _ = hm.Set("ok.example", []byte("https://ok.example")) + + helper := &Helper{InputsHTTP: hm, StrictProbe: true} + + // confirmed host -> resolved URL + require.Equal(t, "https://ok.example", helper.convertInputToType("ok.example", typeURL, "")) + // probed but non-HTTP hosts -> skipped (no raw fallback) + require.Equal(t, "", helper.convertInputToType("127.0.0.1:3306", typeURL, "")) + require.Equal(t, "", helper.convertInputToType("zombie.example:6379", typeURL, "")) + // already-URL inputs are not gated by probe results + require.Equal(t, "https://direct.example", helper.convertInputToType("https://direct.example", typeURL, "")) + // non-HTTP protocols still transform normally + require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeHostWithOptionalPort, "")) +} + +func TestStrictProbeDisabledKeepsRawFallback(t *testing.T) { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + require.NoError(t, err) + t.Cleanup(func() { _ = hm.Close() }) + + helper := &Helper{InputsHTTP: hm, StrictProbe: false} + require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeURL, "")) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index eefb5d89ec..f781855421 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -216,6 +216,10 @@ type Options struct { LeaveDefaultPorts bool // AutomaticScan enables automatic tech based template execution AutomaticScan bool + // StrictProbe losslessly skips templates that cannot reach their + // target service (web templates on non-HTTP ports, network templates on + // closed ports). + StrictProbe bool // Silent suppresses any extra text and only writes found URLs on screen. Silent bool // Validate validates the templates passed to nuclei. @@ -580,6 +584,7 @@ func (options *Options) Copy() *Options { PerHostRateLimit: options.PerHostRateLimit, LeaveDefaultPorts: options.LeaveDefaultPorts, AutomaticScan: options.AutomaticScan, + StrictProbe: options.StrictProbe, Silent: options.Silent, Validate: options.Validate, NoStrictSyntax: options.NoStrictSyntax, From f7d051a0eb47ed2a631a7ac6ac21a2234923d52b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 11 Aug 2026 13:27:56 +0400 Subject: [PATCH 3/7] scan plan --- internal/runner/host_group.go | 402 +++++++++++++++++++++++++++++ internal/runner/host_group_test.go | 128 +++++++++ internal/runner/reachability.go | 2 +- internal/runner/runner.go | 78 +++++- pkg/core/engine.go | 6 + pkg/core/execute_options.go | 11 +- pkg/core/executors.go | 26 ++ pkg/core/plan/plan.go | 147 +++++++++++ pkg/core/plan/plan_test.go | 120 +++++++++ pkg/protocols/protocols.go | 4 + 10 files changed, 912 insertions(+), 12 deletions(-) create mode 100644 internal/runner/host_group.go create mode 100644 internal/runner/host_group_test.go create mode 100644 pkg/core/plan/plan.go create mode 100644 pkg/core/plan/plan_test.go diff --git a/internal/runner/host_group.go b/internal/runner/host_group.go new file mode 100644 index 0000000000..d41ddfe1ae --- /dev/null +++ b/internal/runner/host_group.go @@ -0,0 +1,402 @@ +package runner + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v3/pkg/input" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" + sliceutil "github.com/projectdiscovery/utils/slice" + syncutil "github.com/projectdiscovery/utils/sync" +) + +const hostGroupProbeWorkers = 100 + +// hostGroup is a set of targets that share the same open-port / HTTP signature. +// Used for savings estimates / logging; execution itself is a single spray pass +// with a per-(template,target) filter derived from the same probe data. +type hostGroup struct { + key string + hosts []*contextargs.MetaInput + openPorts map[string]struct{} + httpOK bool +} + +type hostProbeResult struct { + input *contextargs.MetaInput + openPorts []string + httpOK bool +} + +// hostReachabilityIndex maps each input to its probed reachability so the +// engine can skip impossible template×target pairs in a single Execute pass. +type hostReachabilityIndex struct { + byInput map[string]hostReachability +} + +type hostReachability struct { + openPorts map[string]struct{} + httpOK bool +} + +// Allow reports whether template may produce a finding on input (lossless). +func (idx *hostReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool { + if idx == nil || t == nil || mi == nil { + return true + } + h, ok := idx.byInput[mi.Input] + if !ok { + return true + } + return templateAllowedOnHost(t, h.httpOK, h.openPorts) +} + +func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}) bool { + if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) { + return true + } + if isWebTemplate(t) { + return httpOK + } + ports, ok := tcpNetworkOnlyPorts(t) + if !ok { + return true + } + for _, p := range ports { + if _, open := openPorts[p]; open { + return true + } + } + return false +} + +// estimateTemplateRequests mirrors core.getRequestCount for scheduling math. +func estimateTemplateRequests(tpls []*templates.Template) int { + count := 0 + for _, t := range tpls { + if len(t.Workflows) > 0 { + continue + } + if t.TotalRequests > 0 { + count += t.TotalRequests + continue + } + count++ + } + return count +} + +func hostGroupKey(openPorts []string, httpOK bool) string { + ports := append([]string(nil), openPorts...) + sort.Strings(ports) + http := "nohttp" + if httpOK { + http = "http" + } + if len(ports) == 0 { + return http + "|-" + } + return http + "|" + strings.Join(ports, ",") +} + +func isWebTemplate(t *templates.Template) bool { + return len(t.RequestsHTTP)+len(t.RequestsWithHTTP)+len(t.RequestsHeadless)+len(t.RequestsWebsocket) > 0 && + len(t.RequestsNetwork)+len(t.RequestsWithTCP)+len(t.RequestsDNS)+len(t.RequestsFile)+ + len(t.RequestsSSL)+len(t.RequestsCode)+len(t.RequestsJavascript)+len(t.Workflows) == 0 +} + +func isUniversalTemplate(t *templates.Template) bool { + if t.SelfContained || len(t.Workflows) > 0 { + return true + } + if _, ok := tcpNetworkOnlyPorts(t); ok { + return false + } + if isWebTemplate(t) { + return false + } + return true +} + +// templatesForHostGroup returns templates that can produce findings on this group. +func templatesForHostGroup(all []*templates.Template, g hostGroup) []*templates.Template { + out := make([]*templates.Template, 0, len(all)) + for _, t := range all { + if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) { + continue + } + if templateAllowedOnHost(t, g.httpOK, g.openPorts) { + out = append(out, t) + } + } + return out +} + +func partitionTemplates(all []*templates.Template) (web, network, universal, selfContained []*templates.Template) { + for _, t := range all { + switch { + case t.SelfContained: + selfContained = append(selfContained, t) + case len(t.Workflows) > 0: + universal = append(universal, t) + case isWebTemplate(t): + web = append(web, t) + case isUniversalTemplate(t): + universal = append(universal, t) + default: + network = append(network, t) + } + } + return +} + +// buildHostReachability probes each target once, then returns an index for +// single-pass filtering plus host groups for logging/estimates. +// +// HTTP reachability prefers the already-populated InputHelper.InputsHTTP map +// (from initializeTemplatesHTTPInput) so we do not re-run httpx. +// When probePorts is false, only explicit input ports are recorded (HTTP-only mode). +func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *input.Helper, probePorts bool) (*hostReachabilityIndex, []hostGroup, error) { + if r.inputProvider == nil { + return nil, nil, fmt.Errorf("no input provider") + } + + var portsToProbe []string + if probePorts { + portsMap := portsPopularityFromTemplates(tpls) + portsToProbe = make([]string, 0, len(portsMap)) + for p := range portsMap { + if isNumericPort(p) { + portsToProbe = append(portsToProbe, p) + } + } + portsToProbe = sliceutil.Dedupe(portsToProbe) + sort.Strings(portsToProbe) + } + + var targets []*contextargs.MetaInput + r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { + targets = append(targets, mi.Clone()) + return true + }) + if len(targets) == 0 { + return &hostReachabilityIndex{byInput: map[string]hostReachability{}}, nil, nil + } + + // Only build an httpx client if we still need live HTTP probes. + needLiveHTTPProbe := false + for _, mi := range targets { + if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") { + continue + } + if httpHelper != nil && httpHelper.InputsHTTP != nil { + continue // already probed (hit or miss) during initializeTemplatesHTTPInput + } + needLiveHTTPProbe = true + break + } + + var httpClient *httpx.HTTPX + if needLiveHTTPProbe { + dialers := protocolstate.GetDialersWithId(r.options.ExecutionId) + if dialers != nil { + httpxOptions := httpx.DefaultOptions + if r.options.AliveHttpProxy != "" { + httpxOptions.Proxy = r.options.AliveHttpProxy + } else if r.options.AliveSocksProxy != "" { + httpxOptions.Proxy = r.options.AliveSocksProxy + } + httpxOptions.RetryMax = r.options.Retries + if r.options.Timeout > 0 { + httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second + } + httpxOptions.NetworkPolicy = dialers.NetworkPolicy + if c, err := httpx.New(&httpxOptions); err == nil { + httpClient = c + } + } + } + + workers := hostGroupProbeWorkers + // Fan out one task per host:port so closed-port timeouts do not serialize per host. + type probeJob struct { + hostIdx int + host string + port string + } + var jobs []probeJob + hostMeta := make([]struct { + mi *contextargs.MetaInput + host string + explicitPort string + }, len(targets)) + + for i, mi := range targets { + host, explicitPort := hostAndExplicitPort(mi.Target()) + if host == "" { + host, explicitPort = hostAndExplicitPort(mi.Input) + } + hostMeta[i].mi = mi + hostMeta[i].host = host + hostMeta[i].explicitPort = explicitPort + if !probePorts { + continue + } + for _, p := range portsToProbe { + if explicitPort != "" && p != explicitPort { + continue + } + jobs = append(jobs, probeJob{hostIdx: i, host: host, port: p}) + } + } + + if len(jobs) < workers { + workers = len(jobs) + } + if workers < 1 { + workers = 1 + } + + openByHost := make([]map[string]struct{}, len(targets)) + for i := range openByHost { + openByHost[i] = map[string]struct{}{} + if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) { + openByHost[i][ep] = struct{}{} + } + } + + if len(jobs) > 0 { + swg, err := syncutil.New(syncutil.WithSize(workers)) + if err != nil { + return nil, nil, err + } + var mu sync.Mutex + for _, job := range jobs { + swg.Add() + go func(job probeJob) { + defer swg.Done() + switch r.probe(job.host, job.port) { + case portOpen, portUnknown: + mu.Lock() + openByHost[job.hostIdx][job.port] = struct{}{} + mu.Unlock() + } + }(job) + } + swg.Wait() + } + + results := make([]hostProbeResult, len(targets)) + for i, meta := range hostMeta { + open := openByHost[i] + ports := make([]string, 0, len(open)) + for p := range open { + ports = append(ports, p) + } + sort.Strings(ports) + results[i] = hostProbeResult{ + input: meta.mi, + openPorts: ports, + httpOK: resolveHTTPReachability(meta.mi, open, httpHelper, httpClient), + } + } + + idx := &hostReachabilityIndex{byInput: make(map[string]hostReachability, len(results))} + grouped := map[string]*hostGroup{} + order := make([]string, 0) + for _, res := range results { + if res.input == nil { + continue + } + portsSet := make(map[string]struct{}, len(res.openPorts)) + for _, p := range res.openPorts { + portsSet[p] = struct{}{} + } + idx.byInput[res.input.Input] = hostReachability{ + openPorts: portsSet, + httpOK: res.httpOK, + } + + key := hostGroupKey(res.openPorts, res.httpOK) + g, ok := grouped[key] + if !ok { + g = &hostGroup{ + key: key, + openPorts: portsSet, + httpOK: res.httpOK, + } + grouped[key] = g + order = append(order, key) + } + g.hosts = append(g.hosts, res.input) + } + + out := make([]hostGroup, 0, len(order)) + for _, key := range order { + out = append(out, *grouped[key]) + } + return idx, out, nil +} + +func resolveHTTPReachability(mi *contextargs.MetaInput, open map[string]struct{}, helper *input.Helper, client *httpx.HTTPX) bool { + if mi == nil { + return false + } + if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") { + return true + } + // Reuse httpx results from initializeTemplatesHTTPInput when available. + if helper != nil && helper.InputsHTTP != nil { + if probed, ok := helper.InputsHTTP.Get(mi.Input); ok && len(probed) > 0 { + return true + } + // Map was built for all inputs: absence means not HTTP. + return false + } + if client != nil { + return utils.ProbeURL(mi.Input, client) != "" + } + if _, ok := open["80"]; ok { + return true + } + if _, ok := open["443"]; ok { + return true + } + return false +} + +// estimateGroupedExecutions returns baseline and filtered template×host counts. +func estimateGroupedExecutions(all []*templates.Template, groups []hostGroup, hostCount int) (baseline, filtered int) { + baseline = estimateTemplateRequests(all) * hostCount + _, _, universal, selfContained := partitionTemplates(all) + once := append([]*templates.Template{}, selfContained...) + once = append(once, universal...) + filtered = estimateTemplateRequests(once) * hostCount + for _, g := range groups { + filtered += estimateTemplateRequests(templatesForHostGroup(all, g)) * len(g.hosts) + } + return baseline, filtered +} + +// countReachabilityStats returns concrete network template count and distinct numeric ports to probe. +func countReachabilityStats(tpls []*templates.Template) (concreteNetwork int, portsToProbe int) { + ports := map[string]struct{}{} + for _, t := range tpls { + ps, ok := tcpNetworkOnlyPorts(t) + if !ok { + continue + } + concreteNetwork++ + for _, p := range ps { + ports[p] = struct{}{} + } + } + return concreteNetwork, len(ports) +} diff --git a/internal/runner/host_group_test.go b/internal/runner/host_group_test.go new file mode 100644 index 0000000000..51e9f5b0ad --- /dev/null +++ b/internal/runner/host_group_test.go @@ -0,0 +1,128 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/stretchr/testify/require" +) + +func TestHostGroupKey(t *testing.T) { + // Lexicographic sort: "443" < "80". + require.Equal(t, "http|443,80", hostGroupKey([]string{"443", "80"}, true)) + require.Equal(t, "nohttp|22", hostGroupKey([]string{"22"}, false)) + require.Equal(t, "nohttp|-", hostGroupKey(nil, false)) +} + +func TestTemplatesForHostGroup(t *testing.T) { + web := &templates.Template{ + ID: "web", + RequestsHTTP: []*http.Request{{}}, + } + ftp := &templates.Template{ + ID: "ftp", + RequestsNetwork: []*network.Request{{ + Port: "21", + }}, + } + ssh := &templates.Template{ + ID: "ssh", + RequestsNetwork: []*network.Request{{ + Port: "22", + }}, + } + all := []*templates.Template{web, ftp, ssh} + + webGroup := hostGroup{ + key: "http|80,443", + httpOK: true, + openPorts: map[string]struct{}{"80": {}, "443": {}}, + } + got := templatesForHostGroup(all, webGroup) + require.Len(t, got, 1) + require.Equal(t, "web", got[0].ID) + + ftpGroup := hostGroup{ + key: "nohttp|21", + httpOK: false, + openPorts: map[string]struct{}{"21": {}}, + } + got = templatesForHostGroup(all, ftpGroup) + require.Len(t, got, 1) + require.Equal(t, "ftp", got[0].ID) + + sshGroup := hostGroup{ + key: "nohttp|22", + httpOK: false, + openPorts: map[string]struct{}{"22": {}}, + } + got = templatesForHostGroup(all, sshGroup) + require.Len(t, got, 1) + require.Equal(t, "ssh", got[0].ID) +} + +func TestPartitionTemplates(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}} + self := &templates.Template{ID: "self", SelfContained: true} + dynNet := &templates.Template{ + ID: "dyn", + RequestsNetwork: []*network.Request{{Port: ""}}, + } + + w, n, u, s := partitionTemplates([]*templates.Template{web, ftp, self, dynNet}) + require.Len(t, w, 1) + require.Equal(t, "web", w[0].ID) + require.Len(t, n, 1) + require.Equal(t, "ftp", n[0].ID) + require.Len(t, u, 1) + require.Equal(t, "dyn", u[0].ID) + require.Len(t, s, 1) + require.Equal(t, "self", s[0].ID) +} + +func TestEstimateGroupedVsBaseline(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1} + ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}, TotalRequests: 1} + ssh := &templates.Template{ID: "ssh", RequestsNetwork: []*network.Request{{Port: "22"}}, TotalRequests: 1} + all := []*templates.Template{web, ftp, ssh} + + groups := []hostGroup{ + {key: "http|80", httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1", "h2")}, + {key: "nohttp|21", httpOK: false, openPorts: map[string]struct{}{"21": {}}, hosts: metaInputs("h3")}, + {key: "nohttp|22", httpOK: false, openPorts: map[string]struct{}{"22": {}}, hosts: metaInputs("h4")}, + } + + baseline, filtered := estimateGroupedExecutions(all, groups, 4) + require.Equal(t, 12, baseline) + // web*2 + ftp*1 + ssh*1 = 4 + require.Equal(t, 4, filtered) + require.Equal(t, 8, baseline-filtered) +} + +func TestHostReachabilityAllow(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}} + idx := &hostReachabilityIndex{byInput: map[string]hostReachability{ + "h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}}, + "h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}}, + }} + require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"})) + require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"})) + require.False(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h1"})) + require.True(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h2"})) + require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "unknown"})) // lossless +} + +func metaInputs(hosts ...string) []*contextargs.MetaInput { + out := make([]*contextargs.MetaInput, 0, len(hosts)) + for _, h := range hosts { + mi := contextargs.NewMetaInput() + mi.Input = h + out = append(out, mi) + } + return out +} diff --git a/internal/runner/reachability.go b/internal/runner/reachability.go index 3575c2c3ec..467b35b4d7 100644 --- a/internal/runner/reachability.go +++ b/internal/runner/reachability.go @@ -27,7 +27,7 @@ const ( portUnknown // timeout / filtered — treat as possibly open ) -const reachabilityProbeTimeout = 2 * time.Second +const reachabilityProbeTimeout = 300 * time.Millisecond // strictProbeEnabled reports whether the lossless reachability prune // applies to this run. It is limited to standard target inputs; request-shaped diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d76b24f52c..6e4d674449 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -41,6 +41,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" + scanplan "github.com/projectdiscovery/nuclei/v3/pkg/core/plan" "github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates" fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input" @@ -66,11 +67,13 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml" "github.com/projectdiscovery/retryablehttp-go" ptrutil "github.com/projectdiscovery/utils/ptr" + stringsutil "github.com/projectdiscovery/utils/strings" ) var ( @@ -944,17 +947,76 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) return nil, errors.New("no templates provided for scan") } - // Lossless reachability prune (network layer): drop network(tcp) templates - // whose declared ports are definitively closed on every target. - if r.strictProbeEnabled() { - finalTemplates = r.pruneClosedTCPNetworkTemplates(finalTemplates) - } - - // pass input provider to engine - // TODO: this should be not necessary after r.hmapInputProvider is removed + refactored if r.inputProvider == nil { return nil, errors.New("no input provider found") } + + return r.executeTemplatesWithScanPlan(engine, finalTemplates) +} + +// executeTemplatesWithScanPlan chooses spray strategy and optionally attaches a +// reachability filter, then runs a single Execute pass. Wall clock must match or +// beat baseline spray; probing is skipped when its cost would dominate. +func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplates []*templates.Template) (*atomic.Bool, error) { + hostCount := int(r.inputProvider.Count()) + baselineReqs := estimateTemplateRequests(finalTemplates) * hostCount + concreteNet, portsToProbe := countReachabilityStats(finalTemplates) + + planIn := scanplan.Input{ + Hosts: hostCount, + Templates: len(finalTemplates), + Requests: baselineReqs, + BulkSize: r.options.BulkSize, + Stream: r.options.Stream, + StrictProbe: r.strictProbeEnabled(), + PortsToProbe: portsToProbe, + ConcreteNetworkTemplates: concreteNet, + } + p := scanplan.Decide(planIn) + + // Resolve -ss auto before Execute so the engine placeholder is unused. + if stringsutil.EqualFoldAny(r.options.ScanStrategy, scanstrategy.Auto.String(), "") { + r.options.ScanStrategy = p.Strategy + } + + var httpHelper *input.Helper + if opts := engine.ExecuterOptions(); opts != nil { + httpHelper = opts.InputHelper + } + + if p.BuildReachability { + idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts) + if err != nil { + return nil, errors.Wrap(err, "could not build host reachability index") + } + baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount) + p.ApplyFiltered(filtered) + saved := baseline - filtered + pct := 0.0 + if baseline > 0 { + pct = 100 * float64(saved) / float64(baseline) + } + for _, g := range groups { + r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g))) + } + r.Logger.Info().Msgf( + "host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass", + len(groups), baseline, filtered, saved, pct, + ) + if p.UseReachabilityFilter { + engine.TemplateTargetFilter = idx.Allow + defer func() { engine.TemplateTargetFilter = nil }() + } + } + + if opts := engine.ExecuterOptions(); opts != nil && p.ExpectedRequests > 0 && p.UseReachabilityFilter { + opts.ExpectedRequestsOverride = p.ExpectedRequests + defer func() { opts.ExpectedRequestsOverride = 0 }() + } + + r.Logger.Info().Msgf("scan-plan: strategy=%s filter=%v expected_req=%d reason=%s", + r.options.ScanStrategy, p.UseReachabilityFilter, p.ExpectedRequests, p.Reason) + results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering) return results, nil } diff --git a/pkg/core/engine.go b/pkg/core/engine.go index 0a412b6fcc..b74710f712 100644 --- a/pkg/core/engine.go +++ b/pkg/core/engine.go @@ -4,6 +4,8 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) @@ -21,6 +23,10 @@ type Engine struct { executerOpts *protocols.ExecutorOptions Callback func(*output.ResultEvent) // Executed on results Logger *gologger.Logger + // TemplateTargetFilter, when set, decides whether a template should run on a + // given target. Returning false skips that pair without dialing (used by + // strict-probe reachability). nil means always execute. + TemplateTargetFilter func(template *templates.Template, input *contextargs.MetaInput) bool } // New returns a new Engine instance diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go index 393aabb8d0..1617144b6b 100644 --- a/pkg/core/execute_options.go +++ b/pkg/core/execute_options.go @@ -67,12 +67,17 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ // workflow requests are not counted as they can be conditional // templateList count is user requested templates count (before clustering) // totalReqAfterClustering is total requests count after clustering - e.executerOpts.Progress.Init(target.Count(), len(templatesList), int64(totalReqAfterClustering)) + // ExpectedRequestsOverride (scan planner) replaces the total when a + // reachability filter will skip impossible template×target pairs. + reqTotal := int64(totalReqAfterClustering) + if e.executerOpts.ExpectedRequestsOverride > 0 { + reqTotal = e.executerOpts.ExpectedRequestsOverride + } + e.executerOpts.Progress.Init(target.Count(), len(templatesList), reqTotal) } if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") { - // TODO: this is only a placeholder, auto scan strategy should choose scan strategy - // based on no of hosts , templates , stream and other optimization parameters + // Last-resort fallback: runner scan planner normally resolves auto. e.options.ScanStrategy = scanstrategy.TemplateSpray.String() } diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 8b2c47559a..2eec553649 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -168,6 +168,21 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ return true } + // Reachability filter (strict-probe): skip template×target pairs that + // cannot produce findings. Done here so spray stays a single pass. + if !skip && e.TemplateTargetFilter != nil && !e.TemplateTargetFilter(template, scannedValue) { + skip = true + // When Progress was inited with the filtered total, do not credit + // skips again (would overshoot 100%). Otherwise credit baseline. + if e.executerOpts != nil && e.executerOpts.Progress != nil && e.executerOpts.ExpectedRequestsOverride <= 0 { + reqs := int64(template.TotalRequests) + if reqs <= 0 { + reqs = 1 + } + e.executerOpts.Progress.IncrementFailedRequestsBy(reqs) + } + } + tasks <- task{index: index, skip: skip, value: scannedValue} index++ return true @@ -221,6 +236,17 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t break } + if e.TemplateTargetFilter != nil && !e.TemplateTargetFilter(tpl, target) { + if e.executerOpts != nil && e.executerOpts.Progress != nil && e.executerOpts.ExpectedRequestsOverride <= 0 { + reqs := int64(tpl.TotalRequests) + if reqs <= 0 { + reqs = 1 + } + e.executerOpts.Progress.IncrementFailedRequestsBy(reqs) + } + continue + } + // resize check point - nop if there are no changes wp.RefreshWithConfig(e.GetWorkPoolConfig()) diff --git a/pkg/core/plan/plan.go b/pkg/core/plan/plan.go new file mode 100644 index 0000000000..c24dba4952 --- /dev/null +++ b/pkg/core/plan/plan.go @@ -0,0 +1,147 @@ +package plan + +import ( + "fmt" + + "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" +) + +// Input is the scan shape the planner uses to pick strategy and reachability. +type Input struct { + Hosts int + Templates int + Requests int // template×host request estimate (baseline) + BulkSize int + Stream bool + StrictProbe bool + + // Reachability / probe budget (known before or after probing). + PortsToProbe int + ConcreteNetworkTemplates int + // FilteredRequests, when > 0, is the post-filter request estimate used for Progress.Init. + FilteredRequests int + // GroupCount is distinct host signatures after probing (0 = unknown). + GroupCount int +} + +// Plan is the execution plan for a single scan pass. +type Plan struct { + Strategy string + BuildReachability bool + ProbePorts bool + UseReachabilityFilter bool + ExpectedRequests int64 + Reason string +} + +// Decide chooses spray strategy and whether to spend budget on reachability probing. +// +// Hard bar: never choose work that is expected to regress wall clock vs baseline spray. +// When probe cost would dominate and savings are uncertain, reachability is skipped. +func Decide(in Input) Plan { + p := Plan{ + Strategy: chooseStrategy(in), + ExpectedRequests: int64(max(in.Requests, 0)), + } + + stratReason := strategyReason(in, p.Strategy) + if !in.StrictProbe { + p.Reason = stratReason + "; reachability=off" + return p + } + + reach := decideReachability(in) + p.BuildReachability = reach.build + p.ProbePorts = reach.probePorts + p.UseReachabilityFilter = reach.useFilter + p.Reason = stratReason + "; " + reach.reason + + if in.FilteredRequests > 0 { + p.ExpectedRequests = int64(in.FilteredRequests) + } + return p +} + +func chooseStrategy(in Input) string { + if in.Stream || in.Hosts <= 0 || in.Templates <= 0 { + return scanstrategy.TemplateSpray.String() + } + bulk := in.BulkSize + if bulk <= 0 { + bulk = 25 + } + // Host-spray only when the shape strongly favors per-host locality. + // Conservative default is template-spray (historical auto) so wall clock + // does not regress on small / mixed benches. + if in.Hosts <= bulk && in.Hosts <= 10 && in.Templates >= 10*in.Hosts && in.Requests >= 5000 { + return scanstrategy.HostSpray.String() + } + return scanstrategy.TemplateSpray.String() +} + +func strategyReason(in Input, strategy string) string { + return fmt.Sprintf("strategy=%s hosts=%d templates=%d", strategy, in.Hosts, in.Templates) +} + +type reachabilityDecision struct { + build bool + probePorts bool + useFilter bool + reason string +} + +func decideReachability(in Input) reachabilityDecision { + if in.Hosts <= 0 || in.Templates <= 0 { + return reachabilityDecision{reason: "reachability=skipped-empty"} + } + + probeCost := in.PortsToProbe * in.Hosts + baseline := in.Requests + if baseline <= 0 { + baseline = in.Templates * in.Hosts + } + + // No concrete network ports to probe: HTTP-only index is cheap (reuse httpx map). + if in.PortsToProbe == 0 || in.ConcreteNetworkTemplates == 0 { + return reachabilityDecision{ + build: true, + probePorts: false, + useFilter: true, + reason: "reachability=http-only", + } + } + + // Probe budget: if dialing every port×host costs more than the scan itself + // and we have no signal of a mixed fleet, skip probing to match baseline time. + if probeCost > baseline && in.GroupCount <= 1 { + // GroupCount==0 (unknown) or 1 (homogeneous): do not pay for probes. + if probeCost > max(baseline*2, 500) { + return reachabilityDecision{ + reason: fmt.Sprintf("reachability=skipped-probe-budget cost=%d baseline=%d", probeCost, baseline), + } + } + } + + // After probing, homogeneous fleets still benefit from a cheap filter (no extra dials + // during spray for closed-port templates that global prune may have kept). + return reachabilityDecision{ + build: true, + probePorts: true, + useFilter: true, + reason: fmt.Sprintf("reachability=full probe_cost=%d", probeCost), + } +} + +// ApplyFiltered updates ExpectedRequests once post-probe estimates are known. +func (p *Plan) ApplyFiltered(filtered int) { + if filtered > 0 { + p.ExpectedRequests = int64(filtered) + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/pkg/core/plan/plan_test.go b/pkg/core/plan/plan_test.go new file mode 100644 index 0000000000..d8c5217081 --- /dev/null +++ b/pkg/core/plan/plan_test.go @@ -0,0 +1,120 @@ +package plan + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy" + "github.com/stretchr/testify/require" +) + +func TestDecideStrategyHostSpray(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 200, + Requests: 10000, + BulkSize: 25, + }) + require.Equal(t, scanstrategy.HostSpray.String(), p.Strategy) + require.Contains(t, p.Reason, "strategy=host-spray") +} + +func TestDecideStrategyTemplateSpraySmallBench(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 100, + Requests: 500, + BulkSize: 25, + }) + require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy) +} + +func TestDecideStrategyTemplateSprayManyHosts(t *testing.T) { + p := Decide(Input{ + Hosts: 1000, + Templates: 50, + Requests: 50000, + BulkSize: 25, + }) + require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy) +} + +func TestDecideStrategyStreamDefaultsTemplateSpray(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 100, + Stream: true, + BulkSize: 25, + }) + require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy) +} + +func TestDecideReachabilityOffWithoutStrictProbe(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 60, + Requests: 300, + StrictProbe: false, + PortsToProbe: 3, + }) + require.False(t, p.BuildReachability) + require.False(t, p.UseReachabilityFilter) + require.Contains(t, p.Reason, "reachability=off") +} + +func TestDecideReachabilityHTTPOnly(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 60, + Requests: 300, + StrictProbe: true, + PortsToProbe: 0, + ConcreteNetworkTemplates: 0, + }) + require.True(t, p.BuildReachability) + require.False(t, p.ProbePorts) + require.True(t, p.UseReachabilityFilter) + require.Contains(t, p.Reason, "reachability=http-only") +} + +func TestDecideReachabilityProbeBudgetSkip(t *testing.T) { + p := Decide(Input{ + Hosts: 100, + Templates: 10, + Requests: 1000, + StrictProbe: true, + PortsToProbe: 50, // cost = 5000 > 2*baseline + ConcreteNetworkTemplates: 5, + GroupCount: 0, + }) + require.False(t, p.BuildReachability) + require.False(t, p.UseReachabilityFilter) + require.Contains(t, p.Reason, "reachability=skipped-probe-budget") +} + +func TestDecideReachabilityFull(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 60, + Requests: 300, + StrictProbe: true, + PortsToProbe: 3, // cost = 15 << baseline + ConcreteNetworkTemplates: 30, + }) + require.True(t, p.BuildReachability) + require.True(t, p.ProbePorts) + require.True(t, p.UseReachabilityFilter) + require.Contains(t, p.Reason, "reachability=full") +} + +func TestDecideApplyFiltered(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 60, + Requests: 300, + StrictProbe: true, + FilteredRequests: 105, + PortsToProbe: 3, + ConcreteNetworkTemplates: 30, + }) + require.Equal(t, int64(105), p.ExpectedRequests) +} diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 8f7fb4cf71..81ec4c0ac1 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -153,6 +153,10 @@ type ExecutorOptions struct { CustomFastdialer *fastdialer.Dialer // ClusterMappings stores cluster ID to template IDs mapping during execution ClusterMappings *templateTypes.ClusterMappingsMap + // ExpectedRequestsOverride, when > 0, is used by Progress.Init instead of the + // full templates×hosts product. Set by the scan planner when a reachability + // filter will skip impossible pairs so ETA matches filtered work. + ExpectedRequestsOverride int64 } // todo: centralizing components is not feasible with current clogged architecture From dceb85759ef0d86885cb44c02a16513b5f3fb0df Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 11 Aug 2026 19:51:46 +0400 Subject: [PATCH 4/7] tech filter --- README.md | 1 + cmd/nuclei/main.go | 1 + internal/runner/host_group.go | 80 +++++-- internal/runner/host_group_test.go | 83 ++++++- internal/runner/reachability.go | 11 +- internal/runner/reachability_test.go | 19 ++ internal/runner/runner.go | 90 +++++++- internal/runner/tech_filter.go | 209 ++++++++++++++++++ pkg/core/plan/plan.go | 42 +++- pkg/core/plan/plan_test.go | 85 ++++++- pkg/core/techfilter/techfilter.go | 185 ++++++++++++++++ pkg/core/techfilter/techfilter_test.go | 86 +++++++ .../common/protocolstate/memoizer.go | 6 +- pkg/protocols/http/httprespcache/cache.go | 147 ++++++++++++ .../http/httprespcache/cache_test.go | 70 ++++++ pkg/protocols/http/request.go | 20 ++ pkg/protocols/protocols.go | 11 + pkg/templates/cluster.go | 32 ++- pkg/types/types.go | 10 +- 19 files changed, 1129 insertions(+), 59 deletions(-) create mode 100644 internal/runner/tech_filter.go create mode 100644 pkg/core/techfilter/techfilter.go create mode 100644 pkg/core/techfilter/techfilter_test.go create mode 100644 pkg/protocols/http/httprespcache/cache.go create mode 100644 pkg/protocols/http/httprespcache/cache_test.go diff --git a/README.md b/README.md index c81b7f7160..a2647fe4ce 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,7 @@ OPTIMIZATIONS: -irt, -input-read-timeout value timeout on input read (default 3m0s) -nh, -no-httpx disable httpx probing for non-url input -stp, -strict-probe skip templates whose target service is unreachable (HTTP on non-web ports, network on closed ports; lossless) + -tf, -tech-filter opt-in: fingerprint HTTP targets and skip unmatched product/macro-tagged templates (off by default) -preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default) -no-stdin disable stdin processing diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index e794f0614b..062803782a 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -290,6 +290,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions), flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"), flagSet.BoolVarP(&options.StrictProbe, "strict-probe", "stp", false, "skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)"), + flagSet.BoolVarP(&options.TechFilter, "tech-filter", "tf", false, "opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"), diff --git a/internal/runner/host_group.go b/internal/runner/host_group.go index d41ddfe1ae..8bae23ef01 100644 --- a/internal/runner/host_group.go +++ b/internal/runner/host_group.go @@ -13,7 +13,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/utils" - sliceutil "github.com/projectdiscovery/utils/slice" syncutil "github.com/projectdiscovery/utils/sync" ) @@ -23,16 +22,18 @@ const hostGroupProbeWorkers = 100 // Used for savings estimates / logging; execution itself is a single spray pass // with a per-(template,target) filter derived from the same probe data. type hostGroup struct { - key string - hosts []*contextargs.MetaInput - openPorts map[string]struct{} - httpOK bool + key string + hosts []*contextargs.MetaInput + openPorts map[string]struct{} + httpOK bool + explicitPort string // operator host:port; network dials this, not template defaults } type hostProbeResult struct { - input *contextargs.MetaInput - openPorts []string - httpOK bool + input *contextargs.MetaInput + openPorts []string + httpOK bool + explicitPort string } // hostReachabilityIndex maps each input to its probed reachability so the @@ -44,6 +45,10 @@ type hostReachabilityIndex struct { type hostReachability struct { openPorts map[string]struct{} httpOK bool + // explicitPort is the operator-specified port from host:port input (if any). + // Network execution keeps this port (UseNetworkPort does not replace bare + // host:port), so reachability must not require the template's default port. + explicitPort string } // Allow reports whether template may produce a finding on input (lossless). @@ -55,10 +60,10 @@ func (idx *hostReachabilityIndex) Allow(t *templates.Template, mi *contextargs.M if !ok { return true } - return templateAllowedOnHost(t, h.httpOK, h.openPorts) + return templateAllowedOnHost(t, h.httpOK, h.openPorts, h.explicitPort) } -func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}) bool { +func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}, explicitPort string) bool { if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) { return true } @@ -69,6 +74,15 @@ func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[str if !ok { return true } + // host:port inputs: dial the operator-chosen port at execution time, not the + // template default (see contextargs.UseNetworkPort). If that port is + // reachable, network templates may run against it. + if explicitPort != "" { + if _, open := openPorts[explicitPort]; open { + return true + } + return false + } for _, p := range ports { if _, open := openPorts[p]; open { return true @@ -132,7 +146,7 @@ func templatesForHostGroup(all []*templates.Template, g hostGroup) []*templates. if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) { continue } - if templateAllowedOnHost(t, g.httpOK, g.openPorts) { + if templateAllowedOnHost(t, g.httpOK, g.openPorts, g.explicitPort) { out = append(out, t) } } @@ -170,14 +184,24 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i var portsToProbe []string if probePorts { - portsMap := portsPopularityFromTemplates(tpls) - portsToProbe = make([]string, 0, len(portsMap)) - for p := range portsMap { - if isNumericPort(p) { + // Only concrete network-template ports. Do not pull 80/443 from HTTP + // templates via portsPopularityFromTemplates — httpOK already comes from + // scheme / InputsHTTP, and spraying web ports across every host inflates + // probe cost and flakes under parallel dial load. + seen := map[string]struct{}{} + for _, t := range tpls { + ps, ok := tcpNetworkOnlyPorts(t) + if !ok { + continue + } + for _, p := range ps { + if _, dup := seen[p]; dup { + continue + } + seen[p] = struct{}{} portsToProbe = append(portsToProbe, p) } } - portsToProbe = sliceutil.Dedupe(portsToProbe) sort.Strings(portsToProbe) } @@ -302,9 +326,10 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i } sort.Strings(ports) results[i] = hostProbeResult{ - input: meta.mi, - openPorts: ports, - httpOK: resolveHTTPReachability(meta.mi, open, httpHelper, httpClient), + input: meta.mi, + openPorts: ports, + httpOK: resolveHTTPReachability(meta.mi, open, httpHelper, httpClient), + explicitPort: meta.explicitPort, } } @@ -320,17 +345,24 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i portsSet[p] = struct{}{} } idx.byInput[res.input.Input] = hostReachability{ - openPorts: portsSet, - httpOK: res.httpOK, + openPorts: portsSet, + httpOK: res.httpOK, + explicitPort: res.explicitPort, } key := hostGroupKey(res.openPorts, res.httpOK) + if res.explicitPort != "" { + // Keep explicit host:port inputs in their own estimate bucket so + // network-template fan-out is not attributed to bare hosts. + key = key + "|ep:" + res.explicitPort + } g, ok := grouped[key] if !ok { g = &hostGroup{ - key: key, - openPorts: portsSet, - httpOK: res.httpOK, + key: key, + openPorts: portsSet, + httpOK: res.httpOK, + explicitPort: res.explicitPort, } grouped[key] = g order = append(order, key) diff --git a/internal/runner/host_group_test.go b/internal/runner/host_group_test.go index 51e9f5b0ad..889ffc82d4 100644 --- a/internal/runner/host_group_test.go +++ b/internal/runner/host_group_test.go @@ -106,15 +106,94 @@ func TestEstimateGroupedVsBaseline(t *testing.T) { func TestHostReachabilityAllow(t *testing.T) { web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}} + redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}} idx := &hostReachabilityIndex{byInput: map[string]hostReachability{ - "h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}}, - "h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}}, + "h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}}, + "h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}}, + "127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"}, }} require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"})) require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"})) require.False(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h1"})) require.True(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h2"})) require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "unknown"})) // lossless + // Explicit host:port must allow network templates even when template default + // port (6379) differs from the operator-specified port (19637). + require.True(t, idx.Allow(redis, &contextargs.MetaInput{Input: "127.0.0.1:19637"})) + require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "127.0.0.1:19637"})) +} + +func TestTemplateAllowedOnHostUniversal(t *testing.T) { + dnsish := &templates.Template{ID: "dns", RequestsDNS: nil} // no concrete network ports + // force universal via empty network dynamic port + dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}} + require.True(t, templateAllowedOnHost(dyn, false, map[string]struct{}{}, "")) + _ = dnsish + self := &templates.Template{ID: "self", SelfContained: true} + require.True(t, templateAllowedOnHost(self, false, nil, "")) +} + +func TestTemplateAllowedOnHostExplicitPort(t *testing.T) { + redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}} + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + open := map[string]struct{}{"19637": {}} + require.True(t, templateAllowedOnHost(redis, false, open, "19637")) + require.False(t, templateAllowedOnHost(redis, false, open, "")) // bare host: need 6379 + require.False(t, templateAllowedOnHost(web, false, open, "19637")) +} + +func TestTemplatesForHostGroupExplicitPort(t *testing.T) { + redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}} + ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}} + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + g := hostGroup{ + httpOK: false, + openPorts: map[string]struct{}{"19637": {}}, + explicitPort: "19637", + } + got := templatesForHostGroup([]*templates.Template{redis, ftp, web}, g) + require.Len(t, got, 2) + ids := map[string]bool{got[0].ID: true, got[1].ID: true} + require.True(t, ids["redis"] && ids["ftp"]) +} + +func TestCountReachabilityStats(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}} + pg := &templates.Template{ID: "pg", RequestsNetwork: []*network.Request{{Port: "5432,5432"}}} + dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}} + concrete, ports := countReachabilityStats([]*templates.Template{web, redis, pg, dyn}) + require.Equal(t, 2, concrete) + require.Equal(t, 2, ports) // 6379 + 5432 +} + +func TestResolveHTTPReachability(t *testing.T) { + require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "http://web"}, nil, nil, nil)) + require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "https://web"}, nil, nil, nil)) + require.False(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "redis1"}, map[string]struct{}{}, nil, nil)) + require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, nil, nil)) + require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"443": {}}, nil, nil)) +} + +func TestTemplatesForHostGroupSkipsUniversal(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} + univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}} + g := hostGroup{httpOK: true, openPorts: map[string]struct{}{"80": {}}} + got := templatesForHostGroup([]*templates.Template{web, univ}, g) + require.Len(t, got, 1) + require.Equal(t, "web", got[0].ID) +} + +func TestEstimateGroupedIncludesUniversalsOnce(t *testing.T) { + web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1} + univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}, TotalRequests: 1} + groups := []hostGroup{ + {httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1")}, + } + baseline, filtered := estimateGroupedExecutions([]*templates.Template{web, univ}, groups, 1) + require.Equal(t, 2, baseline) + // universal counted once for all hosts + web for http group + require.Equal(t, 2, filtered) } func metaInputs(hosts ...string) []*contextargs.MetaInput { diff --git a/internal/runner/reachability.go b/internal/runner/reachability.go index 467b35b4d7..2414672314 100644 --- a/internal/runner/reachability.go +++ b/internal/runner/reachability.go @@ -27,7 +27,7 @@ const ( portUnknown // timeout / filtered — treat as possibly open ) -const reachabilityProbeTimeout = 300 * time.Millisecond +const reachabilityProbeTimeout = 750 * time.Millisecond // strictProbeEnabled reports whether the lossless reachability prune // applies to this run. It is limited to standard target inputs; request-shaped @@ -277,6 +277,13 @@ func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult if ne, ok := err.(net.Error); ok && ne.Timeout() { return portUnknown } - return portClosed + msg := strings.ToLower(err.Error()) + // Only definitive negatives prune. DNS blips / temporary dialer errors must + // stay unknown so mixed-fleet probing under load remains lossless. + if strings.Contains(msg, "refused") || strings.Contains(msg, "unreachable") || + strings.Contains(msg, "no route to host") || strings.Contains(msg, "network is unreachable") { + return portClosed + } + return portUnknown } diff --git a/internal/runner/reachability_test.go b/internal/runner/reachability_test.go index 3486a1c08b..0dff91cc86 100644 --- a/internal/runner/reachability_test.go +++ b/internal/runner/reachability_test.go @@ -3,6 +3,7 @@ package runner import ( "net" "testing" + "time" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -106,3 +107,21 @@ func TestClassifyDial(t *testing.T) { t.Errorf("classifyDial(closed) = %v, want portClosed", got) } } + +func TestClassifyDialDNSFailureIsUnknown(t *testing.T) { + d := &net.Dialer{} + // NXDOMAIN / resolution failure must not prune (lossless under load). + got := classifyDial(d.DialContext, "no-such-host.invalid:6379", reachabilityProbeTimeout) + if got != portUnknown { + t.Fatalf("classifyDial(dns-fail) = %v, want portUnknown", got) + } +} + +func TestClassifyDialTimeoutIsUnknown(t *testing.T) { + d := &net.Dialer{} + // Blackhole-ish: TEST-NET-1 drop. Short timeout should classify unknown. + got := classifyDial(d.DialContext, "192.0.2.1:65535", 50*time.Millisecond) + if got != portUnknown { + t.Fatalf("classifyDial(timeout) = %v, want portUnknown", got) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 6e4d674449..bd15faa264 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -42,6 +42,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" scanplan "github.com/projectdiscovery/nuclei/v3/pkg/core/plan" + "github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter" "github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates" fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input" @@ -63,6 +64,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -631,6 +633,12 @@ func (r *Runner) RunEnumeration() error { DoNotCache: r.options.DoNotCacheTemplates, Logger: r.Logger, } + // Attach scan-scoped HTTP response cache only for opt-in -tf, and only + // before templates compile so request executers share the same pointer. + // Default scans leave this nil (no behavioral change). + if r.options.TechFilter { + executorOpts.HTTPResponseCache = httprespcache.New() + } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { // Go StdLib style experimental/debug feature switch @@ -955,12 +963,13 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) } // executeTemplatesWithScanPlan chooses spray strategy and optionally attaches a -// reachability filter, then runs a single Execute pass. Wall clock must match or -// beat baseline spray; probing is skipped when its cost would dominate. +// reachability / tech filter, then runs a single Execute pass. Wall clock must +// match or beat baseline spray; probing is skipped when its cost would dominate. func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplates []*templates.Template) (*atomic.Bool, error) { hostCount := int(r.inputProvider.Count()) baselineReqs := estimateTemplateRequests(finalTemplates) * hostCount concreteNet, portsToProbe := countReachabilityStats(finalTemplates) + techBound := techfilter.CountTechBound(finalTemplates) planIn := scanplan.Input{ Hosts: hostCount, @@ -971,6 +980,8 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate StrictProbe: r.strictProbeEnabled(), PortsToProbe: portsToProbe, ConcreteNetworkTemplates: concreteNet, + TechFilter: r.options.TechFilter, + TechBoundTemplates: techBound, } p := scanplan.Decide(planIn) @@ -984,13 +995,18 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate httpHelper = opts.InputHelper } + var reachFilter func(*templates.Template, *contextargs.MetaInput) bool + var techFilterFn func(*templates.Template, *contextargs.MetaInput) bool + var techIdx *techReachabilityIndex + filteredReqs := baselineReqs + if p.BuildReachability { idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts) if err != nil { return nil, errors.Wrap(err, "could not build host reachability index") } baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount) - p.ApplyFiltered(filtered) + filteredReqs = filtered saved := baseline - filtered pct := 0.0 if baseline > 0 { @@ -1004,18 +1020,72 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate len(groups), baseline, filtered, saved, pct, ) if p.UseReachabilityFilter { - engine.TemplateTargetFilter = idx.Allow - defer func() { engine.TemplateTargetFilter = nil }() + reachFilter = idx.Allow } } - if opts := engine.ExecuterOptions(); opts != nil && p.ExpectedRequests > 0 && p.UseReachabilityFilter { - opts.ExpectedRequestsOverride = p.ExpectedRequests - defer func() { opts.ExpectedRequestsOverride = 0 }() + if p.UseTechFilter { + var err error + var cache *httprespcache.Cache + if opts := engine.ExecuterOptions(); opts != nil { + if opts.HTTPResponseCache == nil { + opts.HTTPResponseCache = httprespcache.New() + } + cache = opts.HTTPResponseCache + } + techIdx, err = r.buildTechReachability(finalTemplates, cache) + if err != nil { + return nil, errors.Wrap(err, "could not build tech filter index") + } + base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount) + // When stacked with reachability, take the more aggressive (lower) estimate + // as a lower bound for progress; actual AND filter may skip more. + if techFiltered < filteredReqs { + filteredReqs = techFiltered + } + saved := base - techFiltered + pct := 0.0 + if base > 0 { + pct = 100 * float64(saved) / float64(base) + } + r.Logger.Info().Msgf( + "tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d", + base, techFiltered, saved, pct, techBound, + ) + techFilterFn = techIdx.Allow + } else if opts := engine.ExecuterOptions(); opts != nil && opts.HTTPResponseCache != nil { + // -tf set but planner skipped (e.g. no tech-bound templates): do not + // alter request paths via an idle response cache. + opts.HTTPResponseCache.Disable() + } + + p.ApplyFiltered(filteredReqs) + engine.TemplateTargetFilter = composeTemplateFilters(reachFilter, techFilterFn) + if engine.TemplateTargetFilter != nil { + defer func() { engine.TemplateTargetFilter = nil }() + } + if opts := engine.ExecuterOptions(); opts != nil { + // Cluster members keep their own Info.Tags; apply tech filter there so + // clustering cannot re-introduce skipped product templates. + if techIdx != nil { + opts.ClusterMemberFilter = techIdx.AllowClusterMember + defer func() { opts.ClusterMemberFilter = nil }() + } + if p.ExpectedRequests > 0 && (engine.TemplateTargetFilter != nil || opts.ClusterMemberFilter != nil) { + opts.ExpectedRequestsOverride = p.ExpectedRequests + defer func() { opts.ExpectedRequestsOverride = 0 }() + } + if opts.HTTPResponseCache != nil { + defer func() { + hits, misses, stores := opts.HTTPResponseCache.Stats() + r.Logger.Info().Msgf("http-response-cache: hits=%d misses=%d stores=%d", hits, misses, stores) + opts.HTTPResponseCache = nil + }() + } } - r.Logger.Info().Msgf("scan-plan: strategy=%s filter=%v expected_req=%d reason=%s", - r.options.ScanStrategy, p.UseReachabilityFilter, p.ExpectedRequests, p.Reason) + r.Logger.Info().Msgf("scan-plan: strategy=%s reach=%v tech=%v expected_req=%d reason=%s", + r.options.ScanStrategy, p.UseReachabilityFilter, p.UseTechFilter, p.ExpectedRequests, p.Reason) results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering) return results, nil diff --git a/internal/runner/tech_filter.go b/internal/runner/tech_filter.go new file mode 100644 index 0000000000..c5b54971d2 --- /dev/null +++ b/internal/runner/tech_filter.go @@ -0,0 +1,209 @@ +package runner + +import ( + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter" + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/useragent" + syncutil "github.com/projectdiscovery/utils/sync" + unitutils "github.com/projectdiscovery/utils/unit" + wappalyzer "github.com/projectdiscovery/wappalyzergo" +) + +const ( + techFilterWorkers = 50 + techFilterMaxBody = 1 * unitutils.Mega + techFilterHTTPTimeout = 5 * time.Second +) + +// techReachabilityIndex filters template×target pairs by product/macro tags. +type techReachabilityIndex struct { + byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey +} + +func normalizeInputKey(raw string) string { + return strings.TrimRight(strings.TrimSpace(raw), "/") +} + +// Allow is fail-open: missing profile or unbound template => true. +func (idx *techReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool { + if idx == nil || t == nil || mi == nil { + return true + } + profile, ok := idx.lookup(mi.Input) + if !ok { + return true + } + return techfilter.Allow(profile, t) +} + +func (idx *techReachabilityIndex) lookup(raw string) (techfilter.HostProfile, bool) { + if idx == nil || idx.byInput == nil { + return techfilter.HostProfile{}, false + } + if p, ok := idx.byInput[normalizeInputKey(raw)]; ok { + return p, true + } + if p, ok := idx.byInput[raw]; ok { + return p, true + } + return techfilter.HostProfile{}, false +} + +// AllowClusterMember adapts Allow for ClusterExecuter (template ID + info only). +func (idx *techReachabilityIndex) AllowClusterMember(templateID string, info model.Info, mi *contextargs.MetaInput) bool { + stub := &templates.Template{ID: templateID, Info: info} + return idx.Allow(stub, mi) +} + +// buildTechReachability fingerprints HTTP(S) targets and builds a tag index. +// When cache is non-nil, fingerprint GET responses seed it so matching template +// GETs pay no extra RTT. +func (r *Runner) buildTechReachability(tpls []*templates.Template, cache *httprespcache.Cache) (*techReachabilityIndex, error) { + _ = tpls + wapp, err := wappalyzer.New() + if err != nil { + return nil, err + } + httpclient, err := httpclientpool.Get(r.options, &httpclientpool.Configuration{ + DisableCookie: true, + ResponseHeaderTimeout: techFilterHTTPTimeout, + }, "") + if err != nil { + return nil, err + } + + idx := &techReachabilityIndex{byInput: make(map[string]techfilter.HostProfile)} + var mu sync.Mutex + sg, err := syncutil.New(syncutil.WithSize(techFilterWorkers)) + if err != nil { + return nil, err + } + + fingerprinted := 0 + r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { + if mi == nil || mi.Input == "" { + return true + } + if !looksLikeHTTPTarget(mi.Input) { + return true + } + sg.Add() + go func(input *contextargs.MetaInput) { + defer sg.Done() + profile := fingerprintTarget(wapp, httpclient, input.Input, cache) + key := normalizeInputKey(input.Input) + mu.Lock() + idx.byInput[key] = profile + if profile.HasTags() { + fingerprinted++ + } + mu.Unlock() + }(mi) + return true + }) + sg.Wait() + + gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags", fingerprinted, len(idx.byInput)) + return idx, nil +} + +func looksLikeHTTPTarget(raw string) bool { + lower := strings.ToLower(raw) + return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") +} + +func fingerprintTarget(wapp *wappalyzer.Wappalyze, client *retryablehttp.Client, rawURL string, cache *httprespcache.Cache) techfilter.HostProfile { + // Prefer a trailing-slash form that matches typical {{BaseURL}}/ template URLs. + seedURL := rawURL + if !strings.HasSuffix(seedURL, "/") { + seedURL = seedURL + "/" + } + req, err := retryablehttp.NewRequest(http.MethodGet, seedURL, nil) + if err != nil { + return techfilter.HostProfile{} + } + req.Header.Set("User-Agent", useragent.PickRandom().Raw) + resp, err := client.Do(req) + if err != nil { + return techfilter.HostProfile{} + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, techFilterMaxBody)) + if err != nil { + return techfilter.HostProfile{} + } + if cache != nil { + // Seed both the requested URL and the absolute URL the client ended on. + cache.SeedHTTP(http.MethodGet, seedURL, resp, body) + if resp.Request != nil && resp.Request.URL != nil { + cache.SeedHTTP(http.MethodGet, resp.Request.URL.String(), resp, body) + } + } + info := wapp.FingerprintWithInfo(resp.Header, body) + products := make(map[string][]string, len(info)) + for name, app := range info { + products[name] = append([]string(nil), app.Categories...) + } + return techfilter.ProfileFromFingerprint(products) +} + +// estimateTechFilteredExecutions counts template×host pairs that survive the tech filter. +func estimateTechFilteredExecutions(tpls []*templates.Template, idx *techReachabilityIndex, hostCount int) (baseline, filtered int) { + baseline = estimateTemplateRequests(tpls) * hostCount + if idx == nil || len(idx.byInput) == 0 { + return baseline, baseline + } + filtered = 0 + for _, t := range tpls { + reqs := t.TotalRequests + if reqs <= 0 { + reqs = 1 + } + if len(t.Workflows) > 0 { + continue + } + for _, profile := range idx.byInput { + if techfilter.Allow(profile, t) { + filtered += reqs + } + } + missing := hostCount - len(idx.byInput) + if missing > 0 { + filtered += reqs * missing + } + } + return baseline, filtered +} + +// composeTemplateFilters ANDs optional filters; nil filters are ignored. +func composeTemplateFilters(filters ...func(*templates.Template, *contextargs.MetaInput) bool) func(*templates.Template, *contextargs.MetaInput) bool { + var active []func(*templates.Template, *contextargs.MetaInput) bool + for _, f := range filters { + if f != nil { + active = append(active, f) + } + } + if len(active) == 0 { + return nil + } + return func(t *templates.Template, mi *contextargs.MetaInput) bool { + for _, f := range active { + if !f(t, mi) { + return false + } + } + return true + } +} diff --git a/pkg/core/plan/plan.go b/pkg/core/plan/plan.go index c24dba4952..69153b35ff 100644 --- a/pkg/core/plan/plan.go +++ b/pkg/core/plan/plan.go @@ -22,6 +22,11 @@ type Input struct { FilteredRequests int // GroupCount is distinct host signatures after probing (0 = unknown). GroupCount int + + // TechFilter enables product/macro tag filtering after cheap HTTP fingerprinting. + TechFilter bool + // TechBoundTemplates is how many templates carry product/macro tags. + TechBoundTemplates int } // Plan is the execution plan for a single scan pass. @@ -30,6 +35,7 @@ type Plan struct { BuildReachability bool ProbePorts bool UseReachabilityFilter bool + UseTechFilter bool ExpectedRequests int64 Reason string } @@ -45,16 +51,18 @@ func Decide(in Input) Plan { } stratReason := strategyReason(in, p.Strategy) - if !in.StrictProbe { - p.Reason = stratReason + "; reachability=off" - return p + reachReason := "reachability=off" + if in.StrictProbe { + reach := decideReachability(in) + p.BuildReachability = reach.build + p.ProbePorts = reach.probePorts + p.UseReachabilityFilter = reach.useFilter + reachReason = reach.reason } - reach := decideReachability(in) - p.BuildReachability = reach.build - p.ProbePorts = reach.probePorts - p.UseReachabilityFilter = reach.useFilter - p.Reason = stratReason + "; " + reach.reason + tech := decideTechFilter(in) + p.UseTechFilter = tech.use + p.Reason = stratReason + "; " + reachReason + "; " + tech.reason if in.FilteredRequests > 0 { p.ExpectedRequests = int64(in.FilteredRequests) @@ -132,6 +140,24 @@ func decideReachability(in Input) reachabilityDecision { } } +type techFilterDecision struct { + use bool + reason string +} + +func decideTechFilter(in Input) techFilterDecision { + // Opt-in only: never enable from strategy/reachability alone. + if !in.TechFilter { + return techFilterDecision{reason: "tech-filter=off"} + } + // Skip fingerprinting when nothing is tech-bound — otherwise we add + // probe latency with zero savings (wall-clock regression). + if in.TechBoundTemplates <= 0 { + return techFilterDecision{reason: "tech-filter=skipped-no-bound-templates"} + } + return techFilterDecision{use: true, reason: fmt.Sprintf("tech-filter=on bound=%d", in.TechBoundTemplates)} +} + // ApplyFiltered updates ExpectedRequests once post-probe estimates are known. func (p *Plan) ApplyFiltered(filtered int) { if filtered > 0 { diff --git a/pkg/core/plan/plan_test.go b/pkg/core/plan/plan_test.go index d8c5217081..a97046d7a9 100644 --- a/pkg/core/plan/plan_test.go +++ b/pkg/core/plan/plan_test.go @@ -106,15 +106,84 @@ func TestDecideReachabilityFull(t *testing.T) { require.Contains(t, p.Reason, "reachability=full") } -func TestDecideApplyFiltered(t *testing.T) { +func TestDecideReachabilityEmptyInput(t *testing.T) { + p := Decide(Input{StrictProbe: true, Hosts: 0, Templates: 10}) + require.False(t, p.BuildReachability) + require.Contains(t, p.Reason, "reachability=skipped-empty") +} + +func TestDecideReachabilityConcreteZeroUsesHTTPOnly(t *testing.T) { p := Decide(Input{ - Hosts: 5, - Templates: 60, - Requests: 300, + Hosts: 20, + Templates: 100, + Requests: 2000, StrictProbe: true, - FilteredRequests: 105, - PortsToProbe: 3, - ConcreteNetworkTemplates: 30, + PortsToProbe: 5, + ConcreteNetworkTemplates: 0, // web-only set + }) + require.True(t, p.BuildReachability) + require.False(t, p.ProbePorts) + require.True(t, p.UseReachabilityFilter) +} + +func TestDecideHostSprayRequiresLargeRequestBudget(t *testing.T) { + // hosts small + many templates but requests below threshold → template-spray + p := Decide(Input{ + Hosts: 5, + Templates: 200, + Requests: 4999, + BulkSize: 25, + }) + require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy) +} + +func TestDecideApplyFilteredIgnoredWhenZero(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 10, + Requests: 50, + StrictProbe: false, + FilteredRequests: 0, + }) + require.Equal(t, int64(50), p.ExpectedRequests) + p.ApplyFiltered(0) + require.Equal(t, int64(50), p.ExpectedRequests) + p.ApplyFiltered(12) + require.Equal(t, int64(12), p.ExpectedRequests) +} + +func TestDecideTechFilterOffByDefault(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: false, + TechBoundTemplates: 40, + }) + require.False(t, p.UseTechFilter) + require.Contains(t, p.Reason, "tech-filter=off") +} + +func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: true, + TechBoundTemplates: 0, + }) + require.False(t, p.UseTechFilter) + require.Contains(t, p.Reason, "tech-filter=skipped-no-bound-templates") +} + +func TestDecideTechFilterOn(t *testing.T) { + p := Decide(Input{ + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: true, + TechBoundTemplates: 40, }) - require.Equal(t, int64(105), p.ExpectedRequests) + require.True(t, p.UseTechFilter) + require.Contains(t, p.Reason, "tech-filter=on") } diff --git a/pkg/core/techfilter/techfilter.go b/pkg/core/techfilter/techfilter.go new file mode 100644 index 0000000000..1557b86827 --- /dev/null +++ b/pkg/core/techfilter/techfilter.go @@ -0,0 +1,185 @@ +// Package techfilter maps Wappalyzer products/categories to template tags and +// decides whether a template can produce a finding on a fingerprinted host. +// +// Fail-open rules (coverage over aggressiveness): +// - templates with no product/macro tags always run +// - hosts with no fingerprints always allow every template +// - non-HTTP templates are not filtered here (reachability owns that) +package techfilter + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v3/pkg/templates" +) + +// genericTags are common nuclei tags that do not bind a template to a product. +var genericTags = map[string]struct{}{ + "cve": {}, "cnvd": {}, "edb": {}, "kev": {}, "vkev": {}, + "misconfig": {}, "exposure": {}, "config": {}, "disclosure": {}, + "tech": {}, "detect": {}, "fingerprint": {}, "favicon": {}, "waf": {}, + "http": {}, "network": {}, "dns": {}, "ssl": {}, "tls": {}, + "websocket": {}, "headless": {}, "file": {}, "code": {}, "javascript": {}, + "workflow": {}, "intrusive": {}, "dos": {}, "fuzz": {}, "token-spray": {}, + "osint": {}, "panel": {}, "login": {}, "auth": {}, "unauth": {}, + "default-login": {}, "vuln": {}, "rce": {}, "lfi": {}, "sqli": {}, + "xss": {}, "ssrf": {}, "xxe": {}, "ssti": {}, "idor": {}, + "critical": {}, "high": {}, "medium": {}, "low": {}, "info": {}, + "unknown": {}, "generic": {}, "misc": {}, "bench": {}, +} + +// productAliases normalize common nuclei / wappalyzer spellings to one token. +var productAliases = map[string]string{ + "wp": "wordpress", + "httpd": "apache", + "apache-http-server": "apache", +} + +// categoryMacros maps Wappalyzer category names to short macro tags. +var categoryMacros = map[string]string{ + "cms": "cms", + "ecommerce": "ecommerce", + "web servers": "webserver", + "web-servers": "webserver", + "web frameworks": "web-framework", + "web-frameworks": "web-framework", + "javascript frameworks": "js-framework", + "javascript-frameworks": "js-framework", + "javascript libraries": "js-library", + "javascript-libraries": "js-library", + "cdn": "cdn", + "databases": "database", + "programming languages": "language", + "programming-languages": "language", + "paas": "paas", + "saas": "saas", + "security": "security", + "caching": "caching", + "miscellaneous": "misc", +} + +// HostProfile is the set of product + macro tags inferred for one target. +type HostProfile struct { + Tags map[string]struct{} // empty / nil => unknown => fail-open +} + +// HasTags reports whether fingerprinting produced at least one tag. +func (h HostProfile) HasTags() bool { + return len(h.Tags) > 0 +} + +// NormalizeProduct turns a Wappalyzer / nuclei product name into a tag token. +func NormalizeProduct(name string) string { + name = strings.TrimSpace(strings.ToLower(name)) + if name == "" { + return "" + } + if i := strings.IndexByte(name, ':'); i > 0 { + name = name[:i] + } + name = strings.ReplaceAll(name, "_", "-") + name = strings.Join(strings.Fields(name), "-") + if alias, ok := productAliases[name]; ok { + return alias + } + return name +} + +// NormalizeCategory turns a Wappalyzer category into a macro tag. +func NormalizeCategory(cat string) string { + key := strings.ToLower(strings.TrimSpace(cat)) + if key == "" { + return "" + } + if m, ok := categoryMacros[key]; ok { + return m + } + return strings.ReplaceAll(key, " ", "-") +} + +// IsGenericTag reports whether tag is not product/macro binding. +func IsGenericTag(tag string) bool { + tag = strings.ToLower(strings.TrimSpace(tag)) + if tag == "" { + return true + } + _, ok := genericTags[tag] + return ok +} + +// ProfileFromFingerprint builds a host profile from products and their categories. +// products maps product name -> category names from Wappalyzer. +func ProfileFromFingerprint(products map[string][]string) HostProfile { + tags := make(map[string]struct{}) + for product, cats := range products { + if p := NormalizeProduct(product); p != "" { + tags[p] = struct{}{} + } + for _, c := range cats { + if m := NormalizeCategory(c); m != "" { + tags[m] = struct{}{} + } + } + } + return HostProfile{Tags: tags} +} + +// TemplateProductTags returns product/macro tags that bind a template to a stack. +// Empty means the template is generic and must always be allowed. +func TemplateProductTags(t *templates.Template) []string { + if t == nil { + return nil + } + raw := t.Info.Tags.ToSlice() + out := make([]string, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, tag := range raw { + tag = NormalizeProduct(tag) + if tag == "" || IsGenericTag(tag) { + continue + } + if _, ok := seen[tag]; ok { + continue + } + seen[tag] = struct{}{} + out = append(out, tag) + } + return out +} + +// IsTechBound reports whether the template is constrained to a product/macro. +func IsTechBound(t *templates.Template) bool { + return len(TemplateProductTags(t)) > 0 +} + +// Allow reports whether template may run on a host with the given profile. +// Fail-open when the host was not fingerprinted or the template is unbound. +func Allow(profile HostProfile, t *templates.Template) bool { + if t == nil { + return true + } + bound := TemplateProductTags(t) + if len(bound) == 0 { + return true + } + if !profile.HasTags() { + return true + } + for _, tag := range bound { + if _, ok := profile.Tags[tag]; ok { + return true + } + } + return false +} + +// CountTechBound returns how many templates carry product/macro tags. +func CountTechBound(tpls []*templates.Template) int { + n := 0 + for _, t := range tpls { + if IsTechBound(t) { + n++ + } + } + return n +} diff --git a/pkg/core/techfilter/techfilter_test.go b/pkg/core/techfilter/techfilter_test.go new file mode 100644 index 0000000000..01c6e678e2 --- /dev/null +++ b/pkg/core/techfilter/techfilter_test.go @@ -0,0 +1,86 @@ +package techfilter + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" +) + +func TestNormalizeProductAndCategory(t *testing.T) { + if got := NormalizeProduct("Nginx:1.25"); got != "nginx" { + t.Fatalf("NormalizeProduct = %q", got) + } + if got := NormalizeProduct("wp"); got != "wordpress" { + t.Fatalf("alias = %q", got) + } + if got := NormalizeCategory("Web servers"); got != "webserver" { + t.Fatalf("category = %q", got) + } +} + +func TestProfileFromFingerprint(t *testing.T) { + p := ProfileFromFingerprint(map[string][]string{ + "Nginx": {"Web servers"}, + "WordPress": {"CMS"}, + }) + for _, want := range []string{"nginx", "webserver", "wordpress", "cms"} { + if _, ok := p.Tags[want]; !ok { + t.Fatalf("missing tag %q in %#v", want, p.Tags) + } + } +} + +func tagged(tags ...string) *templates.Template { + return &templates.Template{ + Info: model.Info{Tags: stringslice.StringSlice{Value: tags}}, + } +} + +func TestAllowFailOpenAndMatch(t *testing.T) { + nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}}) + empty := HostProfile{} + + generic := tagged("misconfig", "http") + wp := tagged("cve", "wordpress") + ngx := tagged("nginx", "misconfig") + cms := tagged("cms") + + if !Allow(nginx, generic) { + t.Fatal("generic must always run") + } + if !Allow(empty, wp) { + t.Fatal("unknown host must fail-open for bound templates") + } + if Allow(nginx, wp) { + t.Fatal("wordpress template must not run on nginx-only host") + } + if !Allow(nginx, ngx) { + t.Fatal("nginx template must run on nginx host") + } + if !Allow(nginx, cms) && false { + // cms macro is on nginx? No — nginx profile has webserver, not cms. + } + if Allow(nginx, cms) { + t.Fatal("cms-bound template must not run on nginx-only host") + } + wpHost := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}}) + if !Allow(wpHost, cms) { + t.Fatal("cms macro must match wordpress host") + } + if !Allow(wpHost, wp) { + t.Fatal("wordpress product must match") + } +} + +func TestCountTechBound(t *testing.T) { + n := CountTechBound([]*templates.Template{ + tagged("misconfig"), + tagged("wordpress"), + tagged("nginx", "http"), + }) + if n != 2 { + t.Fatalf("CountTechBound = %d", n) + } +} diff --git a/pkg/protocols/common/protocolstate/memoizer.go b/pkg/protocols/common/protocolstate/memoizer.go index 812472d13e..73218e1d3d 100644 --- a/pkg/protocols/common/protocolstate/memoizer.go +++ b/pkg/protocols/common/protocolstate/memoizer.go @@ -4,11 +4,15 @@ import ( "github.com/projectdiscovery/utils/memoize" ) +// memoizerMaxSize is the process-wide @memo helper cache capacity. +// Keys are xxhash uint64s of the arg string; values are the helper results. +const memoizerMaxSize = 10_000 + var Memoizer *memoize.Memoizer func init() { var err error - Memoizer, err = memoize.New(memoize.WithMaxSize(1500)) + Memoizer, err = memoize.New(memoize.WithMaxSize(memoizerMaxSize)) if err != nil { panic(err) } diff --git a/pkg/protocols/http/httprespcache/cache.go b/pkg/protocols/http/httprespcache/cache.go new file mode 100644 index 0000000000..7421e828a4 --- /dev/null +++ b/pkg/protocols/http/httprespcache/cache.go @@ -0,0 +1,147 @@ +// Package httprespcache is a scan-scoped in-memory cache for safe HTTP GET/HEAD +// responses. It lets tech fingerprinting and templates share the same RTT. +package httprespcache + +import ( + "bytes" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" +) + +// Cache stores response snapshots keyed by METHOD + URL. +type Cache struct { + mu sync.RWMutex + entries map[string]*Entry + disabled atomic.Bool + hits atomic.Uint64 + misses atomic.Uint64 + stores atomic.Uint64 +} + +// Entry is a reusable response snapshot. +type Entry struct { + StatusCode int + Status string + Proto string + ProtoMajor int + ProtoMinor int + Header http.Header + Body []byte + ContentLength int64 +} + +// New returns an empty cache. +func New() *Cache { + return &Cache{entries: make(map[string]*Entry)} +} + +// Disable stops all Get/Set operations (used when -tf is set but the planner +// skips tech filtering, so default scan semantics stay unchanged). +func (c *Cache) Disable() { + if c != nil { + c.disabled.Store(true) + } +} + +// Enabled reports whether the cache is accepting Get/Set. +func (c *Cache) Enabled() bool { + return c != nil && !c.disabled.Load() +} + +// Key builds a cache key for a method and absolute URL. +func Key(method, rawURL string) string { + method = strings.ToUpper(strings.TrimSpace(method)) + rawURL = strings.TrimSpace(rawURL) + return method + " " + rawURL +} + +// KeyFromRequest builds a key from an http.Request. +func KeyFromRequest(req *http.Request) string { + if req == nil || req.URL == nil { + return "" + } + return Key(req.Method, req.URL.String()) +} + +// CacheableRequest reports whether the request is safe to cache/serve. +func CacheableRequest(req *http.Request) bool { + if req == nil || req.URL == nil { + return false + } + method := strings.ToUpper(req.Method) + if method != http.MethodGet && method != http.MethodHead { + return false + } + if req.Body != nil && req.Body != http.NoBody { + return false + } + return true +} + +// Get returns a fresh *http.Response (new Body reader) or nil on miss. +// req is attached as Response.Request so downstream dump/curl code works. +func (c *Cache) Get(key string, req *http.Request) *http.Response { + if !c.Enabled() || key == "" { + return nil + } + c.mu.RLock() + ent, ok := c.entries[key] + c.mu.RUnlock() + if !ok || ent == nil { + c.misses.Add(1) + return nil + } + c.hits.Add(1) + return ent.toResponse(req) +} + +// Set stores a snapshot for key. body should already be fully read. +func (c *Cache) Set(key string, resp *http.Response, body []byte) { + if !c.Enabled() || key == "" || resp == nil { + return + } + ent := &Entry{ + StatusCode: resp.StatusCode, + Status: resp.Status, + Proto: resp.Proto, + ProtoMajor: resp.ProtoMajor, + ProtoMinor: resp.ProtoMinor, + Header: resp.Header.Clone(), + Body: append([]byte(nil), body...), + ContentLength: int64(len(body)), + } + c.mu.Lock() + c.entries[key] = ent + c.mu.Unlock() + c.stores.Add(1) +} + +// SeedHTTP stores a response from an already-buffered fingerprint/probe. +func (c *Cache) SeedHTTP(method, rawURL string, resp *http.Response, body []byte) { + c.Set(Key(method, rawURL), resp, body) +} + +// Stats returns hit/miss/store counters. +func (c *Cache) Stats() (hits, misses, stores uint64) { + if c == nil { + return 0, 0, 0 + } + return c.hits.Load(), c.misses.Load(), c.stores.Load() +} + +func (e *Entry) toResponse(req *http.Request) *http.Response { + return &http.Response{ + Status: e.Status, + StatusCode: e.StatusCode, + Proto: e.Proto, + ProtoMajor: e.ProtoMajor, + ProtoMinor: e.ProtoMinor, + Header: e.Header.Clone(), + Body: io.NopCloser(bytes.NewReader(e.Body)), + ContentLength: e.ContentLength, + Request: req, + } +} diff --git a/pkg/protocols/http/httprespcache/cache_test.go b/pkg/protocols/http/httprespcache/cache_test.go new file mode 100644 index 0000000000..4fe23c98b4 --- /dev/null +++ b/pkg/protocols/http/httprespcache/cache_test.go @@ -0,0 +1,70 @@ +package httprespcache + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCacheRoundTrip(t *testing.T) { + c := New() + req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) + resp := &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Server": []string{"nginx"}}, + Body: io.NopCloser(http.NoBody), + } + c.Set(KeyFromRequest(req), resp, []byte("ok")) + + got := c.Get(KeyFromRequest(req), req) + if got == nil { + t.Fatal("expected cache hit") + } + body, _ := io.ReadAll(got.Body) + if string(body) != "ok" { + t.Fatalf("body=%q", body) + } + if got.Header.Get("Server") != "nginx" { + t.Fatalf("header=%v", got.Header) + } + hits, misses, stores := c.Stats() + if hits != 1 || misses != 0 || stores != 1 { + t.Fatalf("stats hits=%d misses=%d stores=%d", hits, misses, stores) + } + if c.Get(Key(http.MethodGet, "http://example.com/other"), req) != nil { + t.Fatal("expected miss") + } +} + +func TestCacheableRequest(t *testing.T) { + get := httptest.NewRequest(http.MethodGet, "http://x/", nil) + if !CacheableRequest(get) { + t.Fatal("GET should be cacheable") + } + post := httptest.NewRequest(http.MethodPost, "http://x/", nil) + if CacheableRequest(post) { + t.Fatal("POST should not be cacheable") + } +} + +func TestCacheDisabled(t *testing.T) { + c := New() + req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil) + if err != nil { + t.Fatal(err) + } + c.Set(KeyFromRequest(req), &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("x")) + c.Disable() + if c.Get(KeyFromRequest(req), req) != nil { + t.Fatal("disabled cache must miss") + } + c.Set(KeyFromRequest(req), &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("y")) + if c.Enabled() { + t.Fatal("expected disabled") + } +} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index e47c2d8dcd..a77b033e49 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -28,6 +28,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" @@ -840,6 +841,18 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ fromCache = false } } + // Scan-scoped in-memory cache (tech-filter / response reuse). Prefer this + // over a network round-trip for identical safe GET/HEAD requests. + if resp == nil && request.options.HTTPResponseCache != nil && + httprespcache.CacheableRequest(generatedRequest.request.Request) { + if key := httprespcache.KeyFromRequest(generatedRequest.request.Request); key != "" { + if cached := request.options.HTTPResponseCache.Get(key, generatedRequest.request.Request); cached != nil { + resp = cached + fromCache = true + err = nil + } + } + } if resp == nil { if errSignature := request.handleSignature(generatedRequest); errSignature != nil { return errSignature @@ -1014,6 +1027,13 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ errx = errors.Wrap(err, "could not store in project file") } } + if request.options.HTTPResponseCache != nil && !fromCache && + generatedRequest.request != nil && generatedRequest.request.Request != nil && + httprespcache.CacheableRequest(generatedRequest.request.Request) { + if key := httprespcache.KeyFromRequest(generatedRequest.request.Request); key != "" { + request.options.HTTPResponseCache.Set(key, resp, respChain.BodyBytes()) + } + } }) // evaluate responses continuously until first redirect request in reverse order diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 81ec4c0ac1..092ec5c697 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -34,6 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -157,6 +158,13 @@ type ExecutorOptions struct { // full templates×hosts product. Set by the scan planner when a reachability // filter will skip impossible pairs so ETA matches filtered work. ExpectedRequestsOverride int64 + // ClusterMemberFilter, when set, decides whether a clustered template member + // should produce results on a given input. Used so tech/reachability filters + // still apply after request clustering merges templates. + ClusterMemberFilter func(templateID string, info model.Info, input *contextargs.MetaInput) bool + // HTTPResponseCache is an optional scan-scoped in-memory cache for safe + // GET/HEAD responses (shared with tech fingerprinting). + HTTPResponseCache *httprespcache.Cache } // todo: centralizing components is not feasible with current clogged architecture @@ -338,6 +346,9 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions { ExportReqURLPattern: e.ExportReqURLPattern, GlobalMatchers: e.GlobalMatchers, Logger: e.Logger, + ClusterMemberFilter: e.ClusterMemberFilter, + ExpectedRequestsOverride: e.ExpectedRequestsOverride, + HTTPResponseCache: e.HTTPResponseCache, } copy.ClusterMappings = e.ClusterMappings.Copy() copy.CreateTemplateCtxStore() diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 9dfc3f7bb4..61e222b708 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -256,6 +257,12 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { return false, nil } } + + ops := e.operatorsForInput(inputItem.MetaInput) + if len(ops) == 0 { + return false, nil + } + previous := make(map[string]interface{}) dynamicValues := make(map[string]interface{}) @@ -270,7 +277,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if event.InternalEvent == nil { event.InternalEvent = make(map[string]interface{}) } - for _, operator := range e.operators { + for _, operator := range ops { clonedEvent := event.CloneShallow() result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) @@ -297,7 +304,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if !callbackCalled.Load() && e.options.Options.MatcherStatus { // Parse URL fields from the input fields := protocolUtils.GetJsonFieldsFromURL(ctx.Input.MetaInput.Input) - for _, operator := range e.operators { + for _, operator := range ops { errMsg := "" if err != nil { errMsg = err.Error() @@ -334,6 +341,21 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { return results, err } +// operatorsForInput returns cluster members allowed on input by ClusterMemberFilter. +// When the filter is unset, all operators are returned. +func (e *ClusterExecuter) operatorsForInput(mi *contextargs.MetaInput) []*clusteredOperator { + if e.options == nil || e.options.ClusterMemberFilter == nil { + return e.operators + } + out := make([]*clusteredOperator, 0, len(e.operators)) + for _, op := range e.operators { + if e.options.ClusterMemberFilter(op.templateID, op.templateInfo, mi) { + out = append(out, op) + } + } + return out +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { scanCtx := scan.NewScanContext(ctx.Context(), ctx.Input) @@ -345,8 +367,12 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R return nil, nil } } + ops := e.operatorsForInput(inputItem.MetaInput) + if len(ops) == 0 { + return nil, nil + } err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) { - for _, operator := range e.operators { + for _, operator := range ops { clonedEvent := event.CloneShallow() result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) diff --git a/pkg/types/types.go b/pkg/types/types.go index f781855421..7995d8dde6 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -220,6 +220,13 @@ type Options struct { // target service (web templates on non-HTTP ports, network templates on // closed ports). StrictProbe bool + // TechFilter is an opt-in heuristic (CLI: -tf). When false (default), scan + // behavior matches pre-tech-filter nuclei: no fingerprinting, no product-tag + // skipping, no scan-scoped HTTP response cache. When true, fingerprints HTTP + // targets (Wappalyzer) and skips templates whose product/macro tags cannot + // match the host. Fail-open on unknown hosts and unbound templates. No-op + // when no templates are tech-bound. + TechFilter bool // Silent suppresses any extra text and only writes found URLs on screen. Silent bool // Validate validates the templates passed to nuclei. @@ -584,7 +591,8 @@ func (options *Options) Copy() *Options { PerHostRateLimit: options.PerHostRateLimit, LeaveDefaultPorts: options.LeaveDefaultPorts, AutomaticScan: options.AutomaticScan, - StrictProbe: options.StrictProbe, + StrictProbe: options.StrictProbe, + TechFilter: options.TechFilter, Silent: options.Silent, Validate: options.Validate, NoStrictSyntax: options.NoStrictSyntax, From b91ebe9a62498b3ef7a71133d26e0adf46e7bebd Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 11 Aug 2026 20:01:53 +0400 Subject: [PATCH 5/7] fix ci --- internal/runner/reachability.go | 98 ++------------------------ internal/runner/reachability_test.go | 6 +- pkg/core/plan/plan.go | 3 +- pkg/core/techfilter/techfilter_test.go | 3 - 4 files changed, 8 insertions(+), 102 deletions(-) diff --git a/internal/runner/reachability.go b/internal/runner/reachability.go index 2414672314..dbdbc784f1 100644 --- a/internal/runner/reachability.go +++ b/internal/runner/reachability.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" @@ -23,8 +22,8 @@ type probeResult int const ( portClosed probeResult = iota // connection refused / host unreachable - portOpen // connect succeeded - portUnknown // timeout / filtered — treat as possibly open + portOpen // connect succeeded + portUnknown // timeout / filtered — treat as possibly open ) const reachabilityProbeTimeout = 750 * time.Millisecond @@ -52,8 +51,9 @@ func (r *Runner) strictProbeEnabled() bool { // policy (the probe dials only through the policy-configured httpx client). // Conservative: an explicit http/https target, any scheme that responds, or the // inability to probe safely all keep web templates in place. +// Skipped when -no-httpx is set so we do not probe or prune against user intent. func (r *Runner) noHTTPServiceReachable() bool { - if r.inputProvider == nil { + if r.inputProvider == nil || r.options.DisableHTTPProbe { return false } dialers := protocolstate.GetDialersWithId(r.options.ExecutionId) @@ -91,95 +91,6 @@ func (r *Runner) noHTTPServiceReachable() bool { return !anyHTTP } -// pruneClosedTCPNetworkTemplates removes network(TCP) templates whose declared -// port(s) are definitively closed on every target. Provably lossless: such a -// template cannot connect anywhere, so it cannot produce a finding. -// -// Strictly gated to stay lossless: -// - only applies when EVERY target is a bare host (no explicit port); an -// explicit input port overrides the template port, so we skip those. -// - only single-protocol, network-only templates (a mixed template could have -// a reachable request in another protocol). -// - only TCP: any udp:// address disqualifies the template (a TCP probe says -// nothing about a UDP service — e.g. SNMP/mDNS). -// - only concrete numeric ports; empty/dynamic/service-name ports are kept. -// - prunes only when a port is CLOSED (refused); open or indeterminate keeps it. -func (r *Runner) pruneClosedTCPNetworkTemplates(in []*templates.Template) []*templates.Template { - if r.inputProvider == nil || len(in) == 0 { - return in - } - // Collect hosts; bail out (keep everything) if any input carries a port. - var hosts []string - bareOnly := true - r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { - host, port := hostAndExplicitPort(mi.Input) - if port != "" { - bareOnly = false - return false - } - if host != "" { - hosts = append(hosts, host) - } - return true - }) - if !bareOnly || len(hosts) == 0 { - return in - } - hosts = sliceutil.Dedupe(hosts) - - // Identify prunable candidates and the ports to probe. - candidatePorts := map[int][]string{} - toProbe := map[string]struct{}{} - for i, t := range in { - ports, ok := tcpNetworkOnlyPorts(t) - if !ok { - continue - } - candidatePorts[i] = ports - for _, p := range ports { - toProbe[p] = struct{}{} - } - } - if len(candidatePorts) == 0 { - return in - } - - // A port is "reachable" if open or indeterminate on ANY host. - reachable := map[string]bool{} - for p := range toProbe { - for _, h := range hosts { - res := r.probe(h, p) - if res == portOpen || res == portUnknown { - reachable[p] = true - break - } - } - } - - out := in[:0] - pruned := 0 - for i, t := range in { - if ports, ok := candidatePorts[i]; ok { - anyReachable := false - for _, p := range ports { - if reachable[p] { - anyReachable = true - break - } - } - if !anyReachable { - pruned++ - continue - } - } - out = append(out, t) - } - if pruned > 0 { - gologger.Info().Msgf("reachability prune: excluded %d network(tcp) template[s] targeting only closed ports (lossless)", pruned) - } - return out -} - // tcpNetworkOnlyPorts returns the concrete numeric TCP ports of a single-protocol // network template, or ok=false if the template is ineligible for port pruning. func tcpNetworkOnlyPorts(t *templates.Template) ([]string, bool) { @@ -286,4 +197,3 @@ func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult } return portUnknown } - diff --git a/internal/runner/reachability_test.go b/internal/runner/reachability_test.go index 0dff91cc86..c8a638a57f 100644 --- a/internal/runner/reachability_test.go +++ b/internal/runner/reachability_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" - "github.com/projectdiscovery/nuclei/v3/pkg/templates" httpproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" jsproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" ) func TestHostAndExplicitPort(t *testing.T) { @@ -97,7 +97,7 @@ func TestClassifyDial(t *testing.T) { if err != nil { t.Skip("cannot listen:", err) } - defer ln.Close() + defer func() { _ = ln.Close() }() d := &net.Dialer{} if got := classifyDial(d.DialContext, ln.Addr().String(), reachabilityProbeTimeout); got != portOpen { t.Errorf("classifyDial(open) = %v, want portOpen", got) diff --git a/pkg/core/plan/plan.go b/pkg/core/plan/plan.go index 69153b35ff..85c74cbe29 100644 --- a/pkg/core/plan/plan.go +++ b/pkg/core/plan/plan.go @@ -50,7 +50,6 @@ func Decide(in Input) Plan { ExpectedRequests: int64(max(in.Requests, 0)), } - stratReason := strategyReason(in, p.Strategy) reachReason := "reachability=off" if in.StrictProbe { reach := decideReachability(in) @@ -62,7 +61,7 @@ func Decide(in Input) Plan { tech := decideTechFilter(in) p.UseTechFilter = tech.use - p.Reason = stratReason + "; " + reachReason + "; " + tech.reason + p.Reason = strategyReason(in, p.Strategy) + "; " + reachReason + "; " + tech.reason if in.FilteredRequests > 0 { p.ExpectedRequests = int64(in.FilteredRequests) diff --git a/pkg/core/techfilter/techfilter_test.go b/pkg/core/techfilter/techfilter_test.go index 01c6e678e2..3042e476e5 100644 --- a/pkg/core/techfilter/techfilter_test.go +++ b/pkg/core/techfilter/techfilter_test.go @@ -59,9 +59,6 @@ func TestAllowFailOpenAndMatch(t *testing.T) { if !Allow(nginx, ngx) { t.Fatal("nginx template must run on nginx host") } - if !Allow(nginx, cms) && false { - // cms macro is on nginx? No — nginx profile has webserver, not cms. - } if Allow(nginx, cms) { t.Fatal("cms-bound template must not run on nginx-only host") } From 5916da0d0352a1a249f9bc67a29de4e778d2c63b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 11 Aug 2026 20:10:40 +0400 Subject: [PATCH 6/7] harden filters --- internal/runner/host_group.go | 27 ++++-- internal/runner/host_group_test.go | 16 +++- internal/runner/runner.go | 83 ++++++++++--------- internal/runner/tech_filter.go | 60 +++++++++++--- internal/runner/tech_filter_test.go | 21 +++++ pkg/core/techfilter/techfilter.go | 73 ++++++++++++++-- pkg/core/techfilter/techfilter_test.go | 31 +++++++ pkg/input/transform.go | 4 + pkg/protocols/http/httprespcache/cache.go | 75 ++++++++++++++--- .../http/httprespcache/cache_test.go | 47 +++++++++++ pkg/protocols/http/request.go | 4 +- 11 files changed, 362 insertions(+), 79 deletions(-) create mode 100644 internal/runner/tech_filter_test.go diff --git a/internal/runner/host_group.go b/internal/runner/host_group.go index 8bae23ef01..e2de015ea9 100644 --- a/internal/runner/host_group.go +++ b/internal/runner/host_group.go @@ -220,7 +220,7 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") { continue } - if httpHelper != nil && httpHelper.InputsHTTP != nil { + if httpHelper != nil && httpHelper.InputsHTTPProbed { continue // already probed (hit or miss) during initializeTemplatesHTTPInput } needLiveHTTPProbe = true @@ -273,10 +273,13 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i if !probePorts { continue } + // Always probe the operator-specified port (may differ from template + // defaults). Do not spray unrelated template ports onto host:port inputs. + if explicitPort != "" && isNumericPort(explicitPort) { + jobs = append(jobs, probeJob{hostIdx: i, host: host, port: explicitPort}) + continue + } for _, p := range portsToProbe { - if explicitPort != "" && p != explicitPort { - continue - } jobs = append(jobs, probeJob{hostIdx: i, host: host, port: p}) } } @@ -291,8 +294,11 @@ func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *i openByHost := make([]map[string]struct{}, len(targets)) for i := range openByHost { openByHost[i] = map[string]struct{}{} - if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) { - openByHost[i][ep] = struct{}{} + // Only assume explicit ports are open when we are not probing them. + if !probePorts { + if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) { + openByHost[i][ep] = struct{}{} + } } } @@ -389,8 +395,13 @@ func resolveHTTPReachability(mi *contextargs.MetaInput, open map[string]struct{} if probed, ok := helper.InputsHTTP.Get(mi.Input); ok && len(probed) > 0 { return true } - // Map was built for all inputs: absence means not HTTP. - return false + // Empty/skipped probe map (e.g. MultiFormat) must not force non-HTTP. + if !helper.InputsHTTPProbed { + // fall through to live probe / open ports + } else { + // Map was built for all inputs: absence means not HTTP. + return false + } } if client != nil { return utils.ProbeURL(mi.Input, client) != "" diff --git a/internal/runner/host_group_test.go b/internal/runner/host_group_test.go index 889ffc82d4..90b5e50f59 100644 --- a/internal/runner/host_group_test.go +++ b/internal/runner/host_group_test.go @@ -3,6 +3,7 @@ package runner import ( "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/input" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/network" @@ -108,9 +109,9 @@ func TestHostReachabilityAllow(t *testing.T) { ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}} redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}} idx := &hostReachabilityIndex{byInput: map[string]hostReachability{ - "h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}}, - "h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}}, - "127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"}, + "h1": {httpOK: true, openPorts: map[string]struct{}{"80": {}}}, + "h2": {httpOK: false, openPorts: map[string]struct{}{"21": {}}}, + "127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"}, }} require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"})) require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"})) @@ -140,6 +141,8 @@ func TestTemplateAllowedOnHostExplicitPort(t *testing.T) { require.True(t, templateAllowedOnHost(redis, false, open, "19637")) require.False(t, templateAllowedOnHost(redis, false, open, "")) // bare host: need 6379 require.False(t, templateAllowedOnHost(web, false, open, "19637")) + // Closed explicit port: openPorts empty => network templates denied. + require.False(t, templateAllowedOnHost(redis, false, map[string]struct{}{}, "19637")) } func TestTemplatesForHostGroupExplicitPort(t *testing.T) { @@ -175,6 +178,13 @@ func TestResolveHTTPReachability(t *testing.T) { require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"443": {}}, nil, nil)) } +func TestResolveHTTPReachabilitySkippedProbeFailOpen(t *testing.T) { + // MultiFormat leaves an empty InputsHTTP map without probing — must not + // force non-HTTP when open ports still suggest HTTP. + helper := &input.Helper{InputsHTTPProbed: false} + require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, helper, nil)) +} + func TestTemplatesForHostGroupSkipsUniversal(t *testing.T) { web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}} univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index bd15faa264..c99ed3d786 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -791,6 +791,7 @@ func (r *Runner) RunEnumeration() error { return errors.Wrap(err, "could not probe http input") } executorOpts.InputHelper.InputsHTTP = inputHelpers + executorOpts.InputHelper.InputsHTTPProbed = r.inputProvider.InputType() != provider.MultiFormatInputProvider // Under strict-probe, skip web templates per-input for targets that // probed as non-HTTP (lossless), instead of falling back to raw URL. executorOpts.InputHelper.StrictProbe = r.strictProbeEnabled() @@ -981,7 +982,7 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate PortsToProbe: portsToProbe, ConcreteNetworkTemplates: concreteNet, TechFilter: r.options.TechFilter, - TechBoundTemplates: techBound, + TechBoundTemplates: techBound, } p := scanplan.Decide(planIn) @@ -1003,24 +1004,25 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate if p.BuildReachability { idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts) if err != nil { - return nil, errors.Wrap(err, "could not build host reachability index") - } - baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount) - filteredReqs = filtered - saved := baseline - filtered - pct := 0.0 - if baseline > 0 { - pct = 100 * float64(saved) / float64(baseline) - } - for _, g := range groups { - r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g))) - } - r.Logger.Info().Msgf( - "host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass", - len(groups), baseline, filtered, saved, pct, - ) - if p.UseReachabilityFilter { - reachFilter = idx.Allow + r.Logger.Warning().Msgf("could not build host reachability index, continuing unfiltered: %s", err) + } else { + baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount) + filteredReqs = filtered + saved := baseline - filtered + pct := 0.0 + if baseline > 0 { + pct = 100 * float64(saved) / float64(baseline) + } + for _, g := range groups { + r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g))) + } + r.Logger.Info().Msgf( + "host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass", + len(groups), baseline, filtered, saved, pct, + ) + if p.UseReachabilityFilter { + reachFilter = idx.Allow + } } } @@ -1033,26 +1035,31 @@ func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplate } cache = opts.HTTPResponseCache } - techIdx, err = r.buildTechReachability(finalTemplates, cache) + techIdx, err = r.buildTechReachability(finalTemplates, httpHelper, cache) if err != nil { - return nil, errors.Wrap(err, "could not build tech filter index") - } - base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount) - // When stacked with reachability, take the more aggressive (lower) estimate - // as a lower bound for progress; actual AND filter may skip more. - if techFiltered < filteredReqs { - filteredReqs = techFiltered - } - saved := base - techFiltered - pct := 0.0 - if base > 0 { - pct = 100 * float64(saved) / float64(base) - } - r.Logger.Info().Msgf( - "tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d", - base, techFiltered, saved, pct, techBound, - ) - techFilterFn = techIdx.Allow + r.Logger.Warning().Msgf("could not build tech filter index, continuing without tech-filter: %s", err) + if cache != nil { + cache.Disable() + } + techIdx = nil + } else { + base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount) + // When stacked with reachability, take the more aggressive (lower) estimate + // as a lower bound for progress; actual AND filter may skip more. + if techFiltered < filteredReqs { + filteredReqs = techFiltered + } + saved := base - techFiltered + pct := 0.0 + if base > 0 { + pct = 100 * float64(saved) / float64(base) + } + r.Logger.Info().Msgf( + "tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d", + base, techFiltered, saved, pct, techBound, + ) + techFilterFn = techIdx.Allow + } } else if opts := engine.ExecuterOptions(); opts != nil && opts.HTTPResponseCache != nil { // -tf set but planner skipped (e.g. no tech-bound templates): do not // alter request paths via an idle response cache. diff --git a/internal/runner/tech_filter.go b/internal/runner/tech_filter.go index c5b54971d2..e49e0b89b2 100644 --- a/internal/runner/tech_filter.go +++ b/internal/runner/tech_filter.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter" + "github.com/projectdiscovery/nuclei/v3/pkg/input" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" @@ -29,7 +30,8 @@ const ( // techReachabilityIndex filters template×target pairs by product/macro tags. type techReachabilityIndex struct { - byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey + byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey + boundTags map[string][]string // template ID -> product tags } func normalizeInputKey(raw string) string { @@ -45,7 +47,22 @@ func (idx *techReachabilityIndex) Allow(t *templates.Template, mi *contextargs.M if !ok { return true } - return techfilter.Allow(profile, t) + return techfilter.AllowTags(profile, idx.tagsFor(t)) +} + +func (idx *techReachabilityIndex) tagsFor(t *templates.Template) []string { + if t == nil { + return nil + } + if idx.boundTags == nil { + idx.boundTags = make(map[string][]string) + } + if tags, ok := idx.boundTags[t.ID]; ok { + return tags + } + tags := techfilter.TemplateProductTags(t) + idx.boundTags[t.ID] = tags + return tags } func (idx *techReachabilityIndex) lookup(raw string) (techfilter.HostProfile, bool) { @@ -70,7 +87,7 @@ func (idx *techReachabilityIndex) AllowClusterMember(templateID string, info mod // buildTechReachability fingerprints HTTP(S) targets and builds a tag index. // When cache is non-nil, fingerprint GET responses seed it so matching template // GETs pay no extra RTT. -func (r *Runner) buildTechReachability(tpls []*templates.Template, cache *httprespcache.Cache) (*techReachabilityIndex, error) { +func (r *Runner) buildTechReachability(tpls []*templates.Template, httpHelper *input.Helper, cache *httprespcache.Cache) (*techReachabilityIndex, error) { _ = tpls wapp, err := wappalyzer.New() if err != nil { @@ -84,7 +101,10 @@ func (r *Runner) buildTechReachability(tpls []*templates.Template, cache *httpre return nil, err } - idx := &techReachabilityIndex{byInput: make(map[string]techfilter.HostProfile)} + idx := &techReachabilityIndex{ + byInput: make(map[string]techfilter.HostProfile), + boundTags: make(map[string][]string), + } var mu sync.Mutex sg, err := syncutil.New(syncutil.WithSize(techFilterWorkers)) if err != nil { @@ -92,33 +112,50 @@ func (r *Runner) buildTechReachability(tpls []*templates.Template, cache *httpre } fingerprinted := 0 + eligible := 0 r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { if mi == nil || mi.Input == "" { return true } - if !looksLikeHTTPTarget(mi.Input) { + fpURL := fingerprintURLForInput(mi.Input, httpHelper) + if fpURL == "" { return true } + eligible++ sg.Add() - go func(input *contextargs.MetaInput) { + go func(inputKey, url string) { defer sg.Done() - profile := fingerprintTarget(wapp, httpclient, input.Input, cache) - key := normalizeInputKey(input.Input) + profile := fingerprintTarget(wapp, httpclient, url, cache) + key := normalizeInputKey(inputKey) mu.Lock() idx.byInput[key] = profile if profile.HasTags() { fingerprinted++ } mu.Unlock() - }(mi) + }(mi.Input, fpURL) return true }) sg.Wait() - gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags", fingerprinted, len(idx.byInput)) + gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags (eligible=%d)", fingerprinted, len(idx.byInput), eligible) return idx, nil } +// fingerprintURLForInput returns an absolute http(s) URL to fingerprint, or "". +// Bare hosts reuse InputsHTTP probed URLs when available. +func fingerprintURLForInput(raw string, helper *input.Helper) string { + if looksLikeHTTPTarget(raw) { + return raw + } + if helper != nil && helper.InputsHTTP != nil { + if probed, ok := helper.InputsHTTP.Get(raw); ok && len(probed) > 0 { + return string(probed) + } + } + return "" +} + func looksLikeHTTPTarget(raw string) bool { lower := strings.ToLower(raw) return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") @@ -174,8 +211,9 @@ func estimateTechFilteredExecutions(tpls []*templates.Template, idx *techReachab if len(t.Workflows) > 0 { continue } + tags := idx.tagsFor(t) for _, profile := range idx.byInput { - if techfilter.Allow(profile, t) { + if techfilter.AllowTags(profile, tags) { filtered += reqs } } diff --git a/internal/runner/tech_filter_test.go b/internal/runner/tech_filter_test.go new file mode 100644 index 0000000000..30e8a63cc7 --- /dev/null +++ b/internal/runner/tech_filter_test.go @@ -0,0 +1,21 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v3/pkg/input" + "github.com/stretchr/testify/require" +) + +func TestFingerprintURLForInput(t *testing.T) { + require.Equal(t, "http://a/", fingerprintURLForInput("http://a/", nil)) + require.Equal(t, "", fingerprintURLForInput("example.com", nil)) + + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + require.NoError(t, err) + t.Cleanup(func() { _ = hm.Close() }) + require.NoError(t, hm.Set("example.com", []byte("https://example.com"))) + helper := &input.Helper{InputsHTTP: hm, InputsHTTPProbed: true} + require.Equal(t, "https://example.com", fingerprintURLForInput("example.com", helper)) +} diff --git a/pkg/core/techfilter/techfilter.go b/pkg/core/techfilter/techfilter.go index 1557b86827..30fc2d145f 100644 --- a/pkg/core/techfilter/techfilter.go +++ b/pkg/core/techfilter/techfilter.go @@ -4,6 +4,7 @@ // Fail-open rules (coverage over aggressiveness): // - templates with no product/macro tags always run // - hosts with no fingerprints always allow every template +// - CDN/WAF-only fingerprints do not suppress product templates // - non-HTTP templates are not filtered here (reachability owns that) package techfilter @@ -26,6 +27,18 @@ var genericTags = map[string]struct{}{ "xss": {}, "ssrf": {}, "xxe": {}, "ssti": {}, "idor": {}, "critical": {}, "high": {}, "medium": {}, "low": {}, "info": {}, "unknown": {}, "generic": {}, "misc": {}, "bench": {}, + // Meta / source / taxonomy tags that are not products. + "oast": {}, "takeover": {}, "deserialization": {}, "packetstorm": {}, + "seclists": {}, "hackerone": {}, "huntr": {}, "wp-plugin": {}, "wp-theme": {}, + "wordpress-plugin": {}, "wordpress-theme": {}, "authenticated": {}, + "unauthenticated": {}, "bypass": {}, "injection": {}, "traversal": {}, + "redirect": {}, "crlf": {}, "csrf": {}, "cors": {}, +} + +// weakFingerprintTags alone must not suppress product-bound templates +// (e.g. cloudflare/cdn-only fingerprints). +var weakFingerprintTags = map[string]struct{}{ + "cdn": {}, "waf": {}, "misc": {}, "security": {}, "caching": {}, } // productAliases normalize common nuclei / wappalyzer spellings to one token. @@ -68,6 +81,29 @@ func (h HostProfile) HasTags() bool { return len(h.Tags) > 0 } +// HasSubstantiveTags reports whether the profile has a strong category macro +// (cms, webserver, …). CDN/WAF-only fingerprints are not substantive. +func (h HostProfile) HasSubstantiveTags() bool { + for tag := range h.Tags { + if isStrongMacro(tag) { + return true + } + } + return false +} + +func isStrongMacro(tag string) bool { + if _, weak := weakFingerprintTags[tag]; weak { + return false + } + for _, m := range categoryMacros { + if tag == m { + return true + } + } + return false +} + // NormalizeProduct turns a Wappalyzer / nuclei product name into a tag token. func NormalizeProduct(name string) string { name = strings.TrimSpace(strings.ToLower(name)) @@ -103,8 +139,30 @@ func IsGenericTag(tag string) bool { if tag == "" { return true } - _, ok := genericTags[tag] - return ok + if _, ok := genericTags[tag]; ok { + return true + } + // cve2021 / cve-2021 style year tags bind no product. + if strings.HasPrefix(tag, "cve") { + rest := strings.TrimPrefix(tag, "cve") + rest = strings.TrimPrefix(rest, "-") + if rest != "" && isAllDigits(rest) { + return true + } + } + return false +} + +func isAllDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true } // ProfileFromFingerprint builds a host profile from products and their categories. @@ -155,14 +213,15 @@ func IsTechBound(t *templates.Template) bool { // Allow reports whether template may run on a host with the given profile. // Fail-open when the host was not fingerprinted or the template is unbound. func Allow(profile HostProfile, t *templates.Template) bool { - if t == nil { - return true - } - bound := TemplateProductTags(t) + return AllowTags(profile, TemplateProductTags(t)) +} + +// AllowTags is Allow with precomputed product tags (avoids per-pair allocation). +func AllowTags(profile HostProfile, bound []string) bool { if len(bound) == 0 { return true } - if !profile.HasTags() { + if !profile.HasTags() || !profile.HasSubstantiveTags() { return true } for _, tag := range bound { diff --git a/pkg/core/techfilter/techfilter_test.go b/pkg/core/techfilter/techfilter_test.go index 3042e476e5..d88ca325c3 100644 --- a/pkg/core/techfilter/techfilter_test.go +++ b/pkg/core/techfilter/techfilter_test.go @@ -71,11 +71,42 @@ func TestAllowFailOpenAndMatch(t *testing.T) { } } +func TestMetaTagsAreNotTechBound(t *testing.T) { + cases := [][]string{ + {"cve", "cve2021", "oast", "rce"}, + {"takeover", "dns"}, + {"hackerone", "packetstorm", "seclists"}, + {"wp-plugin", "wordpress-plugin"}, + } + for _, tags := range cases { + tpl := tagged(tags...) + if IsTechBound(tpl) { + t.Fatalf("tags %v must not be tech-bound, got %v", tags, TemplateProductTags(tpl)) + } + cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}}) + if !Allow(cdnOnly, tpl) { + t.Fatalf("unbound meta tags %v must run on any host", tags) + } + } +} + +func TestCDNOnlyFingerprintFailOpen(t *testing.T) { + cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}}) + if cdnOnly.HasSubstantiveTags() { + t.Fatalf("cdn-only profile should not be substantive: %#v", cdnOnly.Tags) + } + wp := tagged("wordpress") + if !Allow(cdnOnly, wp) { + t.Fatal("CDN-only fingerprint must fail-open for product templates") + } +} + func TestCountTechBound(t *testing.T) { n := CountTechBound([]*templates.Template{ tagged("misconfig"), tagged("wordpress"), tagged("nginx", "http"), + tagged("cve2023", "oast"), }) if n != 2 { t.Fatalf("CountTechBound = %d", n) diff --git a/pkg/input/transform.go b/pkg/input/transform.go index 2d1ac46e2f..0a32665d79 100644 --- a/pkg/input/transform.go +++ b/pkg/input/transform.go @@ -16,6 +16,10 @@ import ( // Helper is a structure for helping with input transformation type Helper struct { InputsHTTP *hybrid.HybridMap + // InputsHTTPProbed is true when InputsHTTP was filled by httpx probing. + // False means probing was skipped (e.g. MultiFormat) and map absence must + // not be treated as "not HTTP". + InputsHTTPProbed bool // StrictProbe, when set, makes web-template input resolution skip inputs that // were probed and found NOT to be HTTP services (instead of falling back to // a raw URL). Lossless: a non-HTTP port has no HTTP application to find. diff --git a/pkg/protocols/http/httprespcache/cache.go b/pkg/protocols/http/httprespcache/cache.go index 7421e828a4..1569c58fb5 100644 --- a/pkg/protocols/http/httprespcache/cache.go +++ b/pkg/protocols/http/httprespcache/cache.go @@ -11,14 +11,23 @@ import ( "sync/atomic" ) +const ( + defaultMaxEntries = 4096 + defaultMaxBytes = 64 << 20 // 64 MiB of retained response bodies +) + // Cache stores response snapshots keyed by METHOD + URL. type Cache struct { - mu sync.RWMutex - entries map[string]*Entry - disabled atomic.Bool - hits atomic.Uint64 - misses atomic.Uint64 - stores atomic.Uint64 + mu sync.RWMutex + entries map[string]*Entry + disabled atomic.Bool + hits atomic.Uint64 + misses atomic.Uint64 + stores atomic.Uint64 + skipped atomic.Uint64 + maxEntries int + maxBytes int64 + curBytes int64 } // Entry is a reusable response snapshot. @@ -33,9 +42,13 @@ type Entry struct { ContentLength int64 } -// New returns an empty cache. +// New returns an empty cache with default memory bounds. func New() *Cache { - return &Cache{entries: make(map[string]*Entry)} + return &Cache{ + entries: make(map[string]*Entry), + maxEntries: defaultMaxEntries, + maxBytes: defaultMaxBytes, + } } // Disable stops all Get/Set operations (used when -tf is set but the planner @@ -67,6 +80,8 @@ func KeyFromRequest(req *http.Request) string { } // CacheableRequest reports whether the request is safe to cache/serve. +// Requests with representation-changing headers (auth, cookies, Host override, +// custom headers) are excluded so they cannot reuse another context's response. func CacheableRequest(req *http.Request) bool { if req == nil || req.URL == nil { return false @@ -78,6 +93,18 @@ func CacheableRequest(req *http.Request) bool { if req.Body != nil && req.Body != http.NoBody { return false } + if req.Host != "" && req.URL.Host != "" && !strings.EqualFold(req.Host, req.URL.Host) { + return false + } + for k := range req.Header { + switch strings.ToLower(k) { + case "user-agent", "accept", "accept-language", "accept-encoding", + "connection", "upgrade-insecure-requests", "cache-control", "pragma": + continue + default: + return false + } + } return true } @@ -99,10 +126,27 @@ func (c *Cache) Get(key string, req *http.Request) *http.Response { } // Set stores a snapshot for key. body should already be fully read. +// Entries that would exceed the scan-wide budget are skipped. func (c *Cache) Set(key string, resp *http.Response, body []byte) { if !c.Enabled() || key == "" || resp == nil { return } + bodyLen := int64(len(body)) + c.mu.Lock() + defer c.mu.Unlock() + + if old, ok := c.entries[key]; ok { + c.curBytes -= old.ContentLength + delete(c.entries, key) + } else if c.maxEntries > 0 && len(c.entries) >= c.maxEntries { + c.skipped.Add(1) + return + } + if c.maxBytes > 0 && c.curBytes+bodyLen > c.maxBytes { + c.skipped.Add(1) + return + } + ent := &Entry{ StatusCode: resp.StatusCode, Status: resp.Status, @@ -111,11 +155,10 @@ func (c *Cache) Set(key string, resp *http.Response, body []byte) { ProtoMinor: resp.ProtoMinor, Header: resp.Header.Clone(), Body: append([]byte(nil), body...), - ContentLength: int64(len(body)), + ContentLength: bodyLen, } - c.mu.Lock() c.entries[key] = ent - c.mu.Unlock() + c.curBytes += bodyLen c.stores.Add(1) } @@ -132,6 +175,16 @@ func (c *Cache) Stats() (hits, misses, stores uint64) { return c.hits.Load(), c.misses.Load(), c.stores.Load() } +// Len returns the number of cached entries (tests). +func (c *Cache) Len() int { + if c == nil { + return 0 + } + c.mu.RLock() + defer c.mu.RUnlock() + return len(c.entries) +} + func (e *Entry) toResponse(req *http.Request) *http.Response { return &http.Response{ Status: e.Status, diff --git a/pkg/protocols/http/httprespcache/cache_test.go b/pkg/protocols/http/httprespcache/cache_test.go index 4fe23c98b4..b7aadcf383 100644 --- a/pkg/protocols/http/httprespcache/cache_test.go +++ b/pkg/protocols/http/httprespcache/cache_test.go @@ -50,6 +50,21 @@ func TestCacheableRequest(t *testing.T) { if CacheableRequest(post) { t.Fatal("POST should not be cacheable") } + auth := httptest.NewRequest(http.MethodGet, "http://x/", nil) + auth.Header.Set("Authorization", "Bearer t") + if CacheableRequest(auth) { + t.Fatal("Authorization must not be cacheable") + } + cookie := httptest.NewRequest(http.MethodGet, "http://x/", nil) + cookie.Header.Set("Cookie", "a=b") + if CacheableRequest(cookie) { + t.Fatal("Cookie must not be cacheable") + } + hostOverride := httptest.NewRequest(http.MethodGet, "http://x/", nil) + hostOverride.Host = "other.example" + if CacheableRequest(hostOverride) { + t.Fatal("Host override must not be cacheable") + } } func TestCacheDisabled(t *testing.T) { @@ -68,3 +83,35 @@ func TestCacheDisabled(t *testing.T) { t.Fatal("expected disabled") } } + +func TestCacheEntryBudget(t *testing.T) { + c := New() + c.maxEntries = 2 + c.maxBytes = 100 + mk := func(path, body string) { + req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil) + c.Set(KeyFromRequest(req), &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body)) + } + mk("/a", "aa") + mk("/b", "bb") + mk("/c", "cc") // over entry budget + if c.Len() != 2 { + t.Fatalf("len=%d want 2", c.Len()) + } + if c.skipped.Load() == 0 { + t.Fatal("expected skipped store") + } + + c2 := New() + c2.maxEntries = 10 + c2.maxBytes = 5 + mk2 := func(path, body string) { + req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil) + c2.Set(KeyFromRequest(req), &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body)) + } + mk2("/a", "12345") + mk2("/b", "x") // over byte budget + if c2.Len() != 1 { + t.Fatalf("byte budget len=%d want 1", c2.Len()) + } +} diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index a77b033e49..ecfdfc3132 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -28,13 +28,13 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool" @@ -844,6 +844,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // Scan-scoped in-memory cache (tech-filter / response reuse). Prefer this // over a network round-trip for identical safe GET/HEAD requests. if resp == nil && request.options.HTTPResponseCache != nil && + input.CookieJar == nil && httprespcache.CacheableRequest(generatedRequest.request.Request) { if key := httprespcache.KeyFromRequest(generatedRequest.request.Request); key != "" { if cached := request.options.HTTPResponseCache.Get(key, generatedRequest.request.Request); cached != nil { @@ -1028,6 +1029,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } } if request.options.HTTPResponseCache != nil && !fromCache && + input.CookieJar == nil && generatedRequest.request != nil && generatedRequest.request.Request != nil && httprespcache.CacheableRequest(generatedRequest.request.Request) { if key := httprespcache.KeyFromRequest(generatedRequest.request.Request); key != "" { From b6e4a612f7d44f2b7e202e9b7b4525977f5f54a9 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 15 Aug 2026 18:44:01 +0400 Subject: [PATCH 7/7] harden cache --- internal/runner/tech_filter.go | 13 +++-- pkg/core/plan/plan_test.go | 24 ++++----- pkg/protocols/http/httprespcache/cache.go | 51 +++++++++++++++---- .../http/httprespcache/cache_test.go | 33 ++++++++++++ 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/internal/runner/tech_filter.go b/internal/runner/tech_filter.go index e49e0b89b2..d84de59448 100644 --- a/internal/runner/tech_filter.go +++ b/internal/runner/tech_filter.go @@ -182,10 +182,15 @@ func fingerprintTarget(wapp *wappalyzer.Wappalyze, client *retryablehttp.Client, return techfilter.HostProfile{} } if cache != nil { - // Seed both the requested URL and the absolute URL the client ended on. - cache.SeedHTTP(http.MethodGet, seedURL, resp, body) - if resp.Request != nil && resp.Request.URL != nil { - cache.SeedHTTP(http.MethodGet, resp.Request.URL.String(), resp, body) + // Seed under the fingerprint request's full key so matching template + // GETs (same UA / Accept*) reuse this RTT. + cache.SeedHTTP(req.Request, resp, body) + if resp.Request != nil && resp.Request.URL != nil && + resp.Request.URL.String() != seedURL { + finalReq := req.Request.Clone(req.Context()) + finalReq.URL = resp.Request.URL + finalReq.Host = "" + cache.SeedHTTP(finalReq, resp, body) } } info := wapp.FingerprintWithInfo(resp.Header, body) diff --git a/pkg/core/plan/plan_test.go b/pkg/core/plan/plan_test.go index a97046d7a9..60519e8792 100644 --- a/pkg/core/plan/plan_test.go +++ b/pkg/core/plan/plan_test.go @@ -154,10 +154,10 @@ func TestDecideApplyFilteredIgnoredWhenZero(t *testing.T) { func TestDecideTechFilterOffByDefault(t *testing.T) { p := Decide(Input{ - Hosts: 5, - Templates: 100, - Requests: 500, - TechFilter: false, + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: false, TechBoundTemplates: 40, }) require.False(t, p.UseTechFilter) @@ -166,10 +166,10 @@ func TestDecideTechFilterOffByDefault(t *testing.T) { func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) { p := Decide(Input{ - Hosts: 5, - Templates: 100, - Requests: 500, - TechFilter: true, + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: true, TechBoundTemplates: 0, }) require.False(t, p.UseTechFilter) @@ -178,10 +178,10 @@ func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) { func TestDecideTechFilterOn(t *testing.T) { p := Decide(Input{ - Hosts: 5, - Templates: 100, - Requests: 500, - TechFilter: true, + Hosts: 5, + Templates: 100, + Requests: 500, + TechFilter: true, TechBoundTemplates: 40, }) require.True(t, p.UseTechFilter) diff --git a/pkg/protocols/http/httprespcache/cache.go b/pkg/protocols/http/httprespcache/cache.go index 1569c58fb5..434d5df3da 100644 --- a/pkg/protocols/http/httprespcache/cache.go +++ b/pkg/protocols/http/httprespcache/cache.go @@ -6,17 +6,27 @@ import ( "bytes" "io" "net/http" + "sort" "strings" "sync" "sync/atomic" ) +// representationHeaderNames are folded into the cache key so two requests +// that differ only in content-negotiation / UA identity cannot share a body. +var representationHeaderNames = []string{ + "accept", + "accept-encoding", + "accept-language", + "user-agent", +} + const ( defaultMaxEntries = 4096 defaultMaxBytes = 64 << 20 // 64 MiB of retained response bodies ) -// Cache stores response snapshots keyed by METHOD + URL. +// Cache stores response snapshots keyed by METHOD + URL + representation headers. type Cache struct { mu sync.RWMutex entries map[string]*Entry @@ -64,24 +74,38 @@ func (c *Cache) Enabled() bool { return c != nil && !c.disabled.Load() } -// Key builds a cache key for a method and absolute URL. +// Key builds a cache key for a method and absolute URL (no header identity). func Key(method, rawURL string) string { method = strings.ToUpper(strings.TrimSpace(method)) rawURL = strings.TrimSpace(rawURL) return method + " " + rawURL } -// KeyFromRequest builds a key from an http.Request. +// KeyFromRequest builds a key from an http.Request, including representation +// headers (UA / Accept*) so distinct content-negotiation contexts do not collide. func KeyFromRequest(req *http.Request) string { if req == nil || req.URL == nil { return "" } - return Key(req.Method, req.URL.String()) + base := Key(req.Method, req.URL.String()) + parts := make([]string, 0, len(representationHeaderNames)) + for _, name := range representationHeaderNames { + vals := req.Header.Values(name) + if len(vals) == 0 { + continue + } + parts = append(parts, name+":"+strings.Join(vals, ",")) + } + if len(parts) == 0 { + return base + } + sort.Strings(parts) + return base + " |" + strings.Join(parts, "|") } // CacheableRequest reports whether the request is safe to cache/serve. -// Requests with representation-changing headers (auth, cookies, Host override, -// custom headers) are excluded so they cannot reuse another context's response. +// Requests with auth, cookies, Host override, Cache-Control/Pragma, or any +// non-allowlisted header are excluded so they cannot reuse another context. func CacheableRequest(req *http.Request) bool { if req == nil || req.URL == nil { return false @@ -99,8 +123,11 @@ func CacheableRequest(req *http.Request) bool { for k := range req.Header { switch strings.ToLower(k) { case "user-agent", "accept", "accept-language", "accept-encoding", - "connection", "upgrade-insecure-requests", "cache-control", "pragma": + "connection", "upgrade-insecure-requests": continue + case "cache-control", "pragma": + // Explicit cache directives: never serve from the scan cache. + return false default: return false } @@ -162,9 +189,13 @@ func (c *Cache) Set(key string, resp *http.Response, body []byte) { c.stores.Add(1) } -// SeedHTTP stores a response from an already-buffered fingerprint/probe. -func (c *Cache) SeedHTTP(method, rawURL string, resp *http.Response, body []byte) { - c.Set(Key(method, rawURL), resp, body) +// SeedHTTP stores a fingerprint/probe response under the request's full key +// (method + URL + representation headers). +func (c *Cache) SeedHTTP(req *http.Request, resp *http.Response, body []byte) { + if !CacheableRequest(req) { + return + } + c.Set(KeyFromRequest(req), resp, body) } // Stats returns hit/miss/store counters. diff --git a/pkg/protocols/http/httprespcache/cache_test.go b/pkg/protocols/http/httprespcache/cache_test.go index b7aadcf383..cdcd8f0e67 100644 --- a/pkg/protocols/http/httprespcache/cache_test.go +++ b/pkg/protocols/http/httprespcache/cache_test.go @@ -65,6 +65,39 @@ func TestCacheableRequest(t *testing.T) { if CacheableRequest(hostOverride) { t.Fatal("Host override must not be cacheable") } + cc := httptest.NewRequest(http.MethodGet, "http://x/", nil) + cc.Header.Set("Cache-Control", "no-cache") + if CacheableRequest(cc) { + t.Fatal("Cache-Control must not be cacheable") + } + pragma := httptest.NewRequest(http.MethodGet, "http://x/", nil) + pragma.Header.Set("Pragma", "no-cache") + if CacheableRequest(pragma) { + t.Fatal("Pragma must not be cacheable") + } +} + +func TestKeyFromRequestIncludesRepresentationHeaders(t *testing.T) { + a := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) + a.Header.Set("User-Agent", "ua-a") + a.Header.Set("Accept-Language", "en") + + b := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) + b.Header.Set("User-Agent", "ua-a") + b.Header.Set("Accept-Language", "fr") + + if KeyFromRequest(a) == KeyFromRequest(b) { + t.Fatal("distinct Accept-Language must produce distinct keys") + } + + c := New() + c.Set(KeyFromRequest(a), &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("en-body")) + if c.Get(KeyFromRequest(b), b) != nil { + t.Fatal("must not reuse response across Accept-Language") + } + if got := c.Get(KeyFromRequest(a), a); got == nil { + t.Fatal("expected hit for matching headers") + } } func TestCacheDisabled(t *testing.T) {