Added exclude-matchers support for template & matchers (#2218)

* Added exclude-matchers support for template & matchers

* Fixed panics due to typo

* Added support for only template ID + misc cleanup

v2/cmd/nuclei/main.go

@@ -117,6 +117,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.FileNormalizedStringSliceVarP(&options.ExcludeIds, "exclude-id", "eid", []string{}, "templates to exclude based on template ids (comma-separated, file)"),
 		flagSet.FileNormalizedOriginalStringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"),
 		flagSet.FileNormalizedOriginalStringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory to exclude (comma-separated, file)"),
+		flagSet.FileCommaSeparatedStringSliceVarP(&options.ExcludeMatchers, "exclude-matchers", "em", []string{}, "template matchers to exclude in result"),
 		flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
 		flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("templates to run based on protocol type. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),

v2/internal/runner/runner.go

@@ -32,6 +32,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
@@ -333,6 +334,7 @@ func (r *Runner) RunEnumeration() error {
 		HostErrorsCache: cache,
 		Colorizer:       r.colorizer,
 		ResumeCfg:       r.resumeCfg,
+		ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
 	}
 	engine := core.New(r.options)
 	engine.SetExecuterOptions(executerOpts)

v2/pkg/operators/operators.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/sliceutil"
 )
 
@@ -34,6 +35,11 @@ type Operators struct {
 	MatchersCondition string `yaml:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"`
 	// cached variables that may be used along with request.
 	matchersCondition matchers.ConditionType
+
+	// TemplateID is the ID of the template for matcher
+	TemplateID string
+	// ExcludeMatchers is a list of excludeMatchers items
+	ExcludeMatchers *excludematchers.ExcludeMatchers
 }
 
 // Compile compiles the operators as well as their corresponding matchers and extractors
@@ -238,6 +244,12 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 	}
 
 	for matcherIndex, matcher := range operators.Matchers {
+		// Skip matchers that are in the blocklist
+		if operators.ExcludeMatchers != nil {
+			if operators.ExcludeMatchers.Match(operators.TemplateID, matcher.Name) {
+				continue
+			}
+		}
 		if isMatch, matched := match(data, matcher); isMatch {
 			if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled
 				matcherName := getMatcherName(matcher, matcherIndex)

v2/pkg/protocols/common/utils/excludematchers/excludematchers.go

@@ -0,0 +1,67 @@
+package excludematchers
+
+import (
+	"strings"
+)
+
+// ExcludeMatchers is an instance for excluding matchers with template IDs
+type ExcludeMatchers struct {
+	values       map[string]struct{}
+	templateIDs  map[string]struct{}
+	matcherNames map[string]struct{}
+}
+
+// New returns a new exclude matchers instance
+//
+// Wildcard and non-wildcard values are supported.
+// <template-id>:<matcher-name> is the syntax. Wildcards can be specified
+// using * character for either value.
+//
+//  Ex- http-missing-security-headers:* skips all http-missing-security-header templates
+func New(values []string) *ExcludeMatchers {
+	excludeMatchers := &ExcludeMatchers{
+		values:       make(map[string]struct{}),
+		templateIDs:  make(map[string]struct{}),
+		matcherNames: make(map[string]struct{}),
+	}
+	for _, value := range values {
+		partValues := strings.SplitN(value, ":", 2)
+		if len(partValues) < 2 {
+			// If there is no matcher name, consider it as template ID
+			if _, ok := excludeMatchers.templateIDs[value]; !ok {
+				excludeMatchers.templateIDs[value] = struct{}{}
+			}
+			continue
+		}
+		templateID, matcherName := partValues[0], partValues[1]
+
+		// Handle wildcards
+		if templateID == "*" {
+			if _, ok := excludeMatchers.matcherNames[matcherName]; !ok {
+				excludeMatchers.matcherNames[matcherName] = struct{}{}
+			}
+		} else if matcherName == "*" {
+			if _, ok := excludeMatchers.templateIDs[templateID]; !ok {
+				excludeMatchers.templateIDs[templateID] = struct{}{}
+			}
+		} else {
+			if _, ok := excludeMatchers.values[value]; !ok {
+				excludeMatchers.values[value] = struct{}{}
+			}
+		}
+	}
+	return excludeMatchers
+}
+
+// Match returns true if templateID and matcherName matches the blocklist
+func (e *ExcludeMatchers) Match(templateID, matcherName string) bool {
+	if _, ok := e.templateIDs[templateID]; ok {
+		return true
+	}
+	if _, ok := e.matcherNames[matcherName]; ok {
+		return true
+	}
+	matchName := strings.Join([]string{templateID, matcherName}, ":")
+	_, found := e.values[matchName]
+	return found
+}

v2/pkg/protocols/common/utils/excludematchers/excludematchers_test.go

@@ -0,0 +1,19 @@
+package excludematchers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExcludeMatchers(t *testing.T) {
+	em := New([]string{"test-template:test-matcher", "new-template:*", "*:new-matcher", "only-template-id"})
+
+	require.True(t, em.Match("test-template", "test-matcher"), "could not get template-matcher value")
+	require.False(t, em.Match("test-template", "random-matcher"), "could get template-matcher value")
+
+	require.True(t, em.Match("new-template", "random-matcher"), "could not get template-matcher value wildcard")
+	require.True(t, em.Match("random-template", "new-matcher"), "could not get template-matcher value wildcard")
+
+	require.True(t, em.Match("only-template-id", "test"), "could not get only template id match value")
+}

v2/pkg/protocols/dns/dns.go

@@ -130,6 +130,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/dns/operators_test.go

@@ -226,6 +226,10 @@ func TestDNSMakeResult(t *testing.T) {
 	recursion := false
 	testutils.Init(options)
 	templateID := "testing-dns"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		RequestType: DNSRequestTypeHolder{DNSRequestType: A},
 		Class:       "INET",
@@ -246,11 +250,8 @@ func TestDNSMakeResult(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
 

v2/pkg/protocols/dns/request_test.go

@@ -20,6 +20,10 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	recursion := false
 	testutils.Init(options)
 	templateID := "testing-dns"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		RequestType: DNSRequestTypeHolder{DNSRequestType: A},
 		Class:       "INET",
@@ -40,11 +44,8 @@ func TestDNSExecuteWithResults(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile dns request")
 

v2/pkg/protocols/file/file.go

@@ -101,6 +101,8 @@ func (request *Request) GetID() string {
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/file/operators_test.go

@@ -239,6 +239,10 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 
 	testutils.Init(options)
 	templateID := "testing-file"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		ID:          templateID,
 		MaxSize:     "1Gb",
@@ -254,11 +258,8 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 

v2/pkg/protocols/file/request_test.go

@@ -22,6 +22,10 @@ func TestFileExecuteWithResults(t *testing.T) {
 
 	testutils.Init(options)
 	templateID := "testing-file"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
 	request := &Request{
 		ID:          templateID,
 		MaxSize:     "1Gb",
@@ -41,11 +45,8 @@ func TestFileExecuteWithResults(t *testing.T) {
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},
 		},
+		options: executerOpts,
 	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-	})
 	err := request.Compile(executerOpts)
 	require.Nil(t, err, "could not compile file request")
 

v2/pkg/protocols/headless/headless.go

@@ -118,6 +118,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/http/http.go

@@ -270,6 +270,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if compileErr := compiled.Compile(); compileErr != nil {
 			return errors.Wrap(compileErr, "could not compile operators")
 		}

v2/pkg/protocols/network/network.go

@@ -199,6 +199,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/protocols.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
@@ -66,6 +67,8 @@ type ExecuterOptions struct {
 	StopAtFirstMatch bool
 	// Variables is a list of variables from template
 	Variables variables.Variable
+	// ExcludeMatchers is the list of matchers to exclude
+	ExcludeMatchers *excludematchers.ExcludeMatchers
 
 	Operators []*operators.Operators // only used by offlinehttp module
 

v2/pkg/protocols/ssl/ssl.go

@@ -86,6 +86,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/websocket/websocket.go

@@ -110,6 +110,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}

v2/pkg/protocols/whois/whois.go

@@ -64,6 +64,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
+		compiled.ExcludeMatchers = options.ExcludeMatchers
+		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
 			return errors.Wrap(err, "could not compile operators")
 		}