diff --git a/pkg/catalog/index/filter.go b/pkg/catalog/index/filter.go index ac4959a531..fa60d3f4b7 100644 --- a/pkg/catalog/index/filter.go +++ b/pkg/catalog/index/filter.go @@ -4,6 +4,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" @@ -18,6 +19,9 @@ import ( // IncludeTemplates and IncludeTags can force inclusion of templates even if // they match exclusion criteria. type Filter struct { + // once ensures that IDs and ExcludedIDs are compiled the first time Matches is called. + once sync.Once + // Authors to include. Authors []string @@ -30,12 +34,24 @@ type Filter struct { // IncludeTags to force include even if excluded. IncludeTags []string - // IDs to include (supports wildcards, OR logic). + // IDs to include (supports wildcards). IDs []string + // includeIDs is a map of IDs for a quick lookup. + includeIDs map[string]struct{} + + // includeIDPatterns are the patterns extracted from IDs. + includeIDPatterns []string + // ExcludeIDs to exclude (supports wildcards). ExcludeIDs []string + // excludeIDs is a map of ExcludeIDs for a quick lookup. + excludeIDs map[string]struct{} + + // excludeIDPatterns are the patterns extracted from ExcludeIDs. + excludeIDPatterns []string + // IncludeTemplates paths to force include even if excluded. IncludeTemplates []string @@ -57,6 +73,8 @@ type Filter struct { // Matches checks if metadata matches the filter criteria. func (f *Filter) Matches(m *Metadata) bool { + f.once.Do(f.Compile) + if f.isForcedInclude(m) { return true } @@ -108,10 +126,8 @@ func (f *Filter) isExcluded(m *Metadata) bool { } if len(f.ExcludeIDs) > 0 { - for _, excludeID := range f.ExcludeIDs { - if matchesID(m.ID, excludeID) { - return true - } + if f.matchesExcludeID(m.ID) { + return true } } @@ -148,14 +164,7 @@ func (f *Filter) matchesIncludes(m *Metadata) bool { } if len(f.IDs) > 0 { - matched := false - for _, id := range f.IDs { - if matchesID(m.ID, id) { - matched = true - break - } - } - if !matched { + if !f.matchesIncludeID(m.ID) { return false } } @@ -175,19 +184,64 @@ func (f *Filter) matchesIncludes(m *Metadata) bool { return true } -// matchesID checks if template ID matches pattern (supports wildcards). -func matchesID(templateID, pattern string) bool { - // Convert to lowercase for case-insensitive matching +// Compile pre-processes IDs and ExcludeIDs into fast lookup structures. +// The first time Matches is called, this is called automatically. +// If IDs or ExcludeIDs are modified after calling Matches, this must be called manually. +// This method is not thread-safe, so make sure noone is using the filter when calling it. +func (f *Filter) Compile() { + f.includeIDPatterns = nil + f.includeIDs = make(map[string]struct{}, len(f.IDs)) + for _, p := range f.IDs { + idOrPattern := strings.ToLower(p) + if strings.ContainsAny(idOrPattern, "*?[") { + f.includeIDPatterns = append(f.includeIDPatterns, idOrPattern) + } else { + f.includeIDs[idOrPattern] = struct{}{} + } + } + + f.excludeIDPatterns = nil + f.excludeIDs = make(map[string]struct{}, len(f.ExcludeIDs)) + for _, p := range f.ExcludeIDs { + idOrPattern := strings.ToLower(p) + if strings.ContainsAny(idOrPattern, "*?[") { + f.excludeIDPatterns = append(f.excludeIDPatterns, idOrPattern) + } else { + f.excludeIDs[idOrPattern] = struct{}{} + } + } +} + +// matchesIncludeID reports whether templateID matches any entry in includeIDs or includeIDPatterns. +func (f *Filter) matchesIncludeID(templateID string) bool { templateID = strings.ToLower(templateID) - pattern = strings.ToLower(pattern) + if _, ok := f.includeIDs[templateID]; ok { + return true + } + + for _, pattern := range f.includeIDPatterns { + if matched, _ := filepath.Match(pattern, templateID); matched { + return true + } + } + + return false +} - if templateID == pattern { +// matchesExcludeID reports whether templateID matches any entry in excludeIDs or excludeIDPatterns. +func (f *Filter) matchesExcludeID(templateID string) bool { + templateID = strings.ToLower(templateID) + if _, ok := f.excludeIDs[templateID]; ok { return true } - matched, _ := filepath.Match(pattern, templateID) + for _, pattern := range f.excludeIDPatterns { + if matched, _ := filepath.Match(pattern, templateID); matched { + return true + } + } - return matched + return false } // matchesPath checks if template path matches pattern. @@ -318,6 +372,10 @@ func (f *Filter) String() string { parts = append(parts, "ids="+strings.Join(f.IDs, ",")) } + if len(f.ExcludeIDs) > 0 { + parts = append(parts, "exclude-ids="+strings.Join(f.ExcludeIDs, ",")) + } + if len(f.Severities) > 0 { sevs := make([]string, len(f.Severities)) for i, s := range f.Severities { diff --git a/pkg/catalog/index/filter_bench_test.go b/pkg/catalog/index/filter_bench_test.go new file mode 100644 index 0000000000..89641501f1 --- /dev/null +++ b/pkg/catalog/index/filter_bench_test.go @@ -0,0 +1,61 @@ +package index + +import ( + "fmt" + "testing" +) + +const templateCount = 10_000 + +func makeMetadataCorpus(n int) []*Metadata { + corpus := make([]*Metadata, n) + for i := range n { + corpus[i] = &Metadata{ + ID: fmt.Sprintf("cve-2021-%04d", i), + Tags: []string{"cve", "http"}, + Authors: []string{"tester"}, + Severity: "critical", + ProtocolType: "http", + } + } + return corpus +} + +// mixedFilter builds a realistic filter: exactIDs exact IDs (~1/3 match corpus) + wildcardPatterns glob patterns (no match) +func mixedFilter(exactIDs, wildcardPatterns int) *Filter { + ids := make([]string, 0, exactIDs+wildcardPatterns) + for i := range exactIDs { + ids = append(ids, fmt.Sprintf("cve-202%d-%04d", i%3, i)) // ~1/3 hit corpus (year 2021) + } + for i := range wildcardPatterns { + ids = append(ids, fmt.Sprintf("sqli-%04d-*", i)) // wildcard, no match + } + return &Filter{IDs: ids} +} + +func BenchmarkFilterMatches(b *testing.B) { + corpus := makeMetadataCorpus(templateCount) + + cases := []struct{ exact, wildcards int }{ + {10, 0}, + {100, 0}, + {1000, 0}, + {100, 1}, + {100, 3}, + {100, 5}, + } + + for _, tc := range cases { + b.Run(fmt.Sprintf("ids=%d,patterns=%d", tc.exact, tc.wildcards), func(b *testing.B) { + b.ReportAllocs() + + filter := mixedFilter(tc.exact, tc.wildcards) + filter.Compile() + for b.Loop() { + for _, meta := range corpus { + _ = filter.matchesIncludeID(meta.ID) + } + } + }) + } +} diff --git a/pkg/catalog/index/filter_test.go b/pkg/catalog/index/filter_test.go index 2bc4735e6c..5c4bfe5bde 100644 --- a/pkg/catalog/index/filter_test.go +++ b/pkg/catalog/index/filter_test.go @@ -5,9 +5,10 @@ import ( "path/filepath" "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" - "github.com/stretchr/testify/require" ) func TestFilterMatches(t *testing.T) { @@ -196,8 +197,8 @@ func TestMatchesID(t *testing.T) { expected bool }{ {"exact match", "CVE-2021-1234", "CVE-2021-1234", true}, - {"wildcard prefix", "CVE-2021-1234", "CVE-*", true}, - {"wildcard suffix", "CVE-2021-1234", "*-1234", true}, + {"wildcard suffix", "CVE-2021-1234", "CVE-*", true}, + {"wildcard prefix", "CVE-2021-1234", "*-1234", true}, {"wildcard middle", "CVE-2021-1234", "CVE-*-1234", true}, {"no match", "CVE-2021-1234", "CVE-2022-*", false}, {"partial no match", "CVE-2021-1234", "CVE-2021-12", false}, @@ -207,7 +208,9 @@ func TestMatchesID(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := matchesID(tt.id, tt.pattern) + filter := &Filter{IDs: []string{tt.pattern}} + filter.Compile() + result := filter.matchesIncludeID(tt.id) require.Equal(t, tt.expected, result) }) }