perf(catalog): split exact matches & pattern

pkg/catalog/index/filter.go

@@ -4,6 +4,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -18,6 +19,9 @@ import (
 // IncludeTemplates and IncludeTags can force inclusion of templates even if
 // they match exclusion criteria.
 type Filter struct {
+	// once ensures that IDs and ExcludedIDs are compiled the first time Matches is called.
+	once sync.Once
+
 	// Authors to include.
 	Authors []string
 
@@ -30,12 +34,24 @@ type Filter struct {
 	// IncludeTags to force include even if excluded.
 	IncludeTags []string
 
-	// IDs to include (supports wildcards, OR logic).
+	// IDs to include (supports wildcards).
 	IDs []string
 
+	// includeIDs is a map of IDs for a quick lookup.
+	includeIDs map[string]struct{}
+
+	// includeIDPatterns are the patterns extracted from IDs.
+	includeIDPatterns []string
+
 	// ExcludeIDs to exclude (supports wildcards).
 	ExcludeIDs []string
 
+	// excludeIDs is a map of ExcludeIDs for a quick lookup.
+	excludeIDs map[string]struct{}
+
+	// excludeIDPatterns are the patterns extracted from ExcludeIDs.
+	excludeIDPatterns []string
+
 	// IncludeTemplates paths to force include even if excluded.
 	IncludeTemplates []string
 
@@ -57,6 +73,8 @@ type Filter struct {
 
 // Matches checks if metadata matches the filter criteria.
 func (f *Filter) Matches(m *Metadata) bool {
+	f.once.Do(f.Compile)
+
 	if f.isForcedInclude(m) {
 		return true
 	}
@@ -108,10 +126,8 @@ func (f *Filter) isExcluded(m *Metadata) bool {
 	}
 
 	if len(f.ExcludeIDs) > 0 {
-		for _, excludeID := range f.ExcludeIDs {
-			if matchesID(m.ID, excludeID) {
-				return true
-			}
+		if f.matchesExcludeID(m.ID) {
+			return true
 		}
 	}
 
@@ -148,14 +164,7 @@ func (f *Filter) matchesIncludes(m *Metadata) bool {
 	}
 
 	if len(f.IDs) > 0 {
-		matched := false
-		for _, id := range f.IDs {
-			if matchesID(m.ID, id) {
-				matched = true
-				break
-			}
-		}
-		if !matched {
+		if !f.matchesIncludeID(m.ID) {
 			return false
 		}
 	}
@@ -175,19 +184,64 @@ func (f *Filter) matchesIncludes(m *Metadata) bool {
 	return true
 }
 
-// matchesID checks if template ID matches pattern (supports wildcards).
-func matchesID(templateID, pattern string) bool {
-	// Convert to lowercase for case-insensitive matching
+// Compile pre-processes IDs and ExcludeIDs into fast lookup structures.
+// The first time Matches is called, this is called automatically.
+// If IDs or ExcludeIDs are modified after calling Matches, this must be called manually.
+// This method is not thread-safe, so make sure noone is using the filter when calling it.
+func (f *Filter) Compile() {
+	f.includeIDPatterns = nil
+	f.includeIDs = make(map[string]struct{}, len(f.IDs))
+	for _, p := range f.IDs {
+		idOrPattern := strings.ToLower(p)
+		if strings.ContainsAny(idOrPattern, "*?[") {
+			f.includeIDPatterns = append(f.includeIDPatterns, idOrPattern)
+		} else {
+			f.includeIDs[idOrPattern] = struct{}{}
+		}
+	}
+
+	f.excludeIDPatterns = nil
+	f.excludeIDs = make(map[string]struct{}, len(f.ExcludeIDs))
+	for _, p := range f.ExcludeIDs {
+		idOrPattern := strings.ToLower(p)
+		if strings.ContainsAny(idOrPattern, "*?[") {
+			f.excludeIDPatterns = append(f.excludeIDPatterns, idOrPattern)
+		} else {
+			f.excludeIDs[idOrPattern] = struct{}{}
+		}
+	}
+}
+
+// matchesIncludeID reports whether templateID matches any entry in includeIDs or includeIDPatterns.
+func (f *Filter) matchesIncludeID(templateID string) bool {
 	templateID = strings.ToLower(templateID)
-	pattern = strings.ToLower(pattern)
+	if _, ok := f.includeIDs[templateID]; ok {
+		return true
+	}
+
+	for _, pattern := range f.includeIDPatterns {
+		if matched, _ := filepath.Match(pattern, templateID); matched {
+			return true
+		}
+	}
+
+	return false
+}
 
-	if templateID == pattern {
+// matchesExcludeID reports whether templateID matches any entry in excludeIDs or excludeIDPatterns.
+func (f *Filter) matchesExcludeID(templateID string) bool {
+	templateID = strings.ToLower(templateID)
+	if _, ok := f.excludeIDs[templateID]; ok {
 		return true
 	}
 
-	matched, _ := filepath.Match(pattern, templateID)
+	for _, pattern := range f.excludeIDPatterns {
+		if matched, _ := filepath.Match(pattern, templateID); matched {
+			return true
+		}
+	}
 
-	return matched
+	return false
 }
 
 // matchesPath checks if template path matches pattern.
@@ -318,6 +372,10 @@ func (f *Filter) String() string {
 		parts = append(parts, "ids="+strings.Join(f.IDs, ","))
 	}
 
+	if len(f.ExcludeIDs) > 0 {
+		parts = append(parts, "exclude-ids="+strings.Join(f.ExcludeIDs, ","))
+	}
+
 	if len(f.Severities) > 0 {
 		sevs := make([]string, len(f.Severities))
 		for i, s := range f.Severities {

pkg/catalog/index/filter_bench_test.go

@@ -0,0 +1,61 @@
+package index
+
+import (
+	"fmt"
+	"testing"
+)
+
+const templateCount = 10_000
+
+func makeMetadataCorpus(n int) []*Metadata {
+	corpus := make([]*Metadata, n)
+	for i := range n {
+		corpus[i] = &Metadata{
+			ID:           fmt.Sprintf("cve-2021-%04d", i),
+			Tags:         []string{"cve", "http"},
+			Authors:      []string{"tester"},
+			Severity:     "critical",
+			ProtocolType: "http",
+		}
+	}
+	return corpus
+}
+
+// mixedFilter builds a realistic filter: exactIDs exact IDs (~1/3 match corpus) + wildcardPatterns glob patterns (no match)
+func mixedFilter(exactIDs, wildcardPatterns int) *Filter {
+	ids := make([]string, 0, exactIDs+wildcardPatterns)
+	for i := range exactIDs {
+		ids = append(ids, fmt.Sprintf("cve-202%d-%04d", i%3, i)) // ~1/3 hit corpus (year 2021)
+	}
+	for i := range wildcardPatterns {
+		ids = append(ids, fmt.Sprintf("sqli-%04d-*", i)) // wildcard, no match
+	}
+	return &Filter{IDs: ids}
+}
+
+func BenchmarkFilterMatches(b *testing.B) {
+	corpus := makeMetadataCorpus(templateCount)
+
+	cases := []struct{ exact, wildcards int }{
+		{10, 0},
+		{100, 0},
+		{1000, 0},
+		{100, 1},
+		{100, 3},
+		{100, 5},
+	}
+
+	for _, tc := range cases {
+		b.Run(fmt.Sprintf("ids=%d,patterns=%d", tc.exact, tc.wildcards), func(b *testing.B) {
+			b.ReportAllocs()
+
+			filter := mixedFilter(tc.exact, tc.wildcards)
+			filter.Compile()
+			for b.Loop() {
+				for _, meta := range corpus {
+					_ = filter.matchesIncludeID(meta.ID)
+				}
+			}
+		})
+	}
+}

pkg/catalog/index/filter_test.go

@@ -5,9 +5,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
-	"github.com/stretchr/testify/require"
 )
 
 func TestFilterMatches(t *testing.T) {
@@ -196,8 +197,8 @@ func TestMatchesID(t *testing.T) {
 		expected bool
 	}{
 		{"exact match", "CVE-2021-1234", "CVE-2021-1234", true},
-		{"wildcard prefix", "CVE-2021-1234", "CVE-*", true},
-		{"wildcard suffix", "CVE-2021-1234", "*-1234", true},
+		{"wildcard suffix", "CVE-2021-1234", "CVE-*", true},
+		{"wildcard prefix", "CVE-2021-1234", "*-1234", true},
 		{"wildcard middle", "CVE-2021-1234", "CVE-*-1234", true},
 		{"no match", "CVE-2021-1234", "CVE-2022-*", false},
 		{"partial no match", "CVE-2021-1234", "CVE-2021-12", false},
@@ -207,7 +208,9 @@ func TestMatchesID(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := matchesID(tt.id, tt.pattern)
+			filter := &Filter{IDs: []string{tt.pattern}}
+			filter.Compile()
+			result := filter.matchesIncludeID(tt.id)
 			require.Equal(t, tt.expected, result)
 		})
 	}