From 6b66f911474877f0ae28ec79673f2ef3c809b564 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 26 Jul 2026 11:10:07 +0700 Subject: [PATCH 1/3] test(lib): add BenchmarkExecuteNucleiWithOptsCtx Signed-off-by: Dwi Siswanto --- lib/multi_bench_test.go | 73 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 lib/multi_bench_test.go diff --git a/lib/multi_bench_test.go b/lib/multi_bench_test.go new file mode 100644 index 0000000000..f1362ee270 --- /dev/null +++ b/lib/multi_bench_test.go @@ -0,0 +1,73 @@ +package nuclei + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index" +) + +func BenchmarkExecuteNucleiWithOptsCtx(b *testing.B) { + templatePath := filepath.Join(b.TempDir(), "benchmark.yaml") + err := os.WriteFile(templatePath, []byte(`id: benchmark-execute-nuclei-with-opts-ctx + +info: + name: Benchmark ExecuteNucleiWithOptsCtx + author: pdteam + severity: info + +http: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: status + status: + - 418 +`), 0o600) + if err != nil { + b.Fatalf("could not write benchmark template: %s", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + b.Cleanup(server.Close) + + _, err = index.NewIndex(b.TempDir()) + if err != nil { + b.Fatalf("could not create benchmark metadata index: %s", err) + } + + engine, err := NewThreadSafeNucleiEngineCtx( + b.Context(), + DisableUpdateCheck(), + WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}), + ) + if err != nil { + b.Fatalf("could not create thread-safe nuclei engine: %s", err) + } + b.Cleanup(engine.Close) + + // Keep benchmark metadata isolated from the user's persistent cache. + // engine.eng.metadataIndexOnce.Do(func() { + // engine.eng.metadataIndex = metadataIndex + // }) + + targets := []string{server.URL} + if err := engine.ExecuteNucleiWithOptsCtx(b.Context(), targets); err != nil { + b.Fatalf("could not warm up nuclei execution: %s", err) + } + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + if err := engine.ExecuteNucleiWithOptsCtx(b.Context(), targets); err != nil { + b.Fatalf("could not execute nuclei: %s", err) + } + } +} From 6e990151b4770293e7359150f3e66e33eddecbd5 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 26 Jul 2026 11:10:47 +0700 Subject: [PATCH 2/3] test(lib): reuse metadata index in BenchmarkExecuteNucleiWithOptsCtx Signed-off-by: Dwi Siswanto --- lib/multi_bench_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/multi_bench_test.go b/lib/multi_bench_test.go index f1362ee270..6cfff2c35c 100644 --- a/lib/multi_bench_test.go +++ b/lib/multi_bench_test.go @@ -37,7 +37,7 @@ http: })) b.Cleanup(server.Close) - _, err = index.NewIndex(b.TempDir()) + metadataIndex, err := index.NewIndex(b.TempDir()) if err != nil { b.Fatalf("could not create benchmark metadata index: %s", err) } @@ -53,9 +53,9 @@ http: b.Cleanup(engine.Close) // Keep benchmark metadata isolated from the user's persistent cache. - // engine.eng.metadataIndexOnce.Do(func() { - // engine.eng.metadataIndex = metadataIndex - // }) + engine.eng.metadataIndexOnce.Do(func() { + engine.eng.metadataIndex = metadataIndex + }) targets := []string{server.URL} if err := engine.ExecuteNucleiWithOptsCtx(b.Context(), targets); err != nil { From f239d6e7c6b1ecde477f47cdab2c6dc8b2264996 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 26 Jul 2026 11:11:10 +0700 Subject: [PATCH 3/3] feat(lib): reuse metadata cache across thread-safe scans Keep one engine-owned metadata index across SDK calls and persist it at shutdown instead of loading and saving a new index for each execution. Track dirty state and parser validation so compatible metadata can skip redundant filtering parses w/o trusting lax entries in strict mode. Defensively copy cached values, invalidate stale entries synchronously, and remove the unused ephemeral engine allocation. Signed-off-by: Dwi Siswanto --- lib/multi.go | 7 +- lib/sdk.go | 33 ++++- pkg/catalog/index/index.go | 64 ++++++--- pkg/catalog/index/index_test.go | 160 +++++++++++++++++++++ pkg/catalog/index/metadata.go | 41 ++++++ pkg/catalog/loader/loader.go | 134 ++++++++++------- pkg/catalog/loader/loader_test.go | 231 ++++++++++++++++++++++++++++++ 7 files changed, 597 insertions(+), 73 deletions(-) diff --git a/lib/multi.go b/lib/multi.go index 338ed15927..0a2eb6d12d 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -22,7 +22,6 @@ import ( // in ThreadSafeNucleiEngine type unsafeOptions struct { executerOpts *protocols.ExecutorOptions - engine *core.Engine } // createEphemeralObjects creates ephemeral nuclei objects/instances/types @@ -52,8 +51,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types opts.RateLimitDuration = time.Second } u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration) - u.engine = core.New(opts) - u.engine.SetExecuterOptions(u.executerOpts) return u, nil } @@ -145,7 +142,9 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t } unsafeOpts.executerOpts.WorkflowLoader = workflowLoader - store, err := loader.New(loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts)) + loaderConfig := loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts) + loaderConfig.MetadataIndex = e.eng.getMetadataIndex() + store, err := loader.New(loaderConfig) if err != nil { return errkit.Wrapf(err, "Could not create loader client: %s", err) } diff --git a/lib/sdk.go b/lib/sdk.go index 9dd5b598fa..b95e70ea10 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" + metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" @@ -83,6 +84,9 @@ type NucleiEngine struct { ownsParser bool // true when the engine created the parser and may purge it on Close authprovider authprovider.AuthProvider + metadataIndex *metadataindex.Index + metadataIndexOnce sync.Once + // unexported meta options opts *types.Options interactshOpts *interactsh.Options @@ -100,6 +104,24 @@ type NucleiEngine struct { tmpDir string } +func (e *NucleiEngine) getMetadataIndex() *metadataindex.Index { + e.metadataIndexOnce.Do(func() { + metadataIndex, err := metadataindex.NewDefaultIndex() + if err != nil { + e.Logger.Warning().Msgf("Could not create metadata cache: %v", err) + + return + } + + e.metadataIndex = metadataIndex + if err := e.metadataIndex.Load(); err != nil { + e.Logger.Warning().Msgf("Could not load metadata cache: %v", err) + } + }) + + return e.metadataIndex +} + // LoadAllTemplates loads all nuclei template based on given options func (e *NucleiEngine) LoadAllTemplates() error { workflowLoader, err := workflow.NewLoader(e.executerOpts) @@ -108,7 +130,11 @@ func (e *NucleiEngine) LoadAllTemplates() error { } e.executerOpts.WorkflowLoader = workflowLoader - e.store, err = loader.New(loader.NewConfig(e.opts, e.catalog, e.executerOpts)) + loaderConfig := loader.NewConfig(e.opts, e.catalog, e.executerOpts) + if e.mode == threadSafe { + loaderConfig.MetadataIndex = e.getMetadataIndex() + } + e.store, err = loader.New(loaderConfig) if err != nil { return errkit.Wrapf(err, "Could not create loader client: %s", err) } @@ -239,6 +265,11 @@ func (e *NucleiEngine) closeInternal() { if e.httpxClient != nil { _ = e.httpxClient.Close() } + if e.metadataIndex != nil { + if err := e.metadataIndex.Save(); err != nil { + e.Logger.Warning().Msgf("Could not save metadata cache: %v", err) + } + } if e.tmpDir != "" { _ = os.RemoveAll(e.tmpDir) } diff --git a/pkg/catalog/index/index.go b/pkg/catalog/index/index.go index 3024851b72..aec8c742c3 100644 --- a/pkg/catalog/index/index.go +++ b/pkg/catalog/index/index.go @@ -2,6 +2,7 @@ package index import ( "encoding/gob" + "errors" "maps" "os" "path/filepath" @@ -34,6 +35,7 @@ type Index struct { cacheFile string mu sync.RWMutex version int + dirty bool } // cacheSnapshot represents the serialized cache structure. @@ -105,20 +107,26 @@ func NewDefaultIndex() (*Index, error) { // Get retrieves metadata for a template path, validating freshness via mtime. func (i *Index) Get(path string) (*Metadata, bool) { i.mu.RLock() - defer i.mu.RUnlock() - metadata, found := i.cache.GetIfPresent(path) + i.mu.RUnlock() + if !found { return nil, false } - if !metadata.IsValid() { - go i.Delete(path) + if metadata.IsValid() { + return metadata.clone(), true + } - return nil, false + i.mu.Lock() + current, found := i.cache.GetIfPresent(path) + if found && current == metadata { + i.cache.Invalidate(path) + i.dirty = true } + i.mu.Unlock() - return metadata, true + return nil, false } // Set stores metadata for a template path. @@ -132,7 +140,10 @@ func (i *Index) Set(path string, metadata *Metadata) (*Metadata, bool) { i.mu.Lock() defer i.mu.Unlock() - return i.cache.Set(path, metadata) + cached, ok := i.cache.Set(path, metadata.clone()) + i.dirty = true + + return cached.clone(), ok } // SetFromTemplate extracts metadata from a parsed template and stores it. @@ -171,7 +182,12 @@ func (i *Index) Delete(path string) { i.mu.Lock() defer i.mu.Unlock() + if _, found := i.cache.GetIfPresent(path); !found { + return + } + i.cache.Invalidate(path) + i.dirty = true } // Size returns the number of cached entries. @@ -188,18 +204,22 @@ func (i *Index) Clear() { defer i.mu.Unlock() i.cache.InvalidateAll() + i.dirty = true } // Save persists the cache to disk using gob encoding. func (i *Index) Save() error { - i.mu.RLock() - defer i.mu.RUnlock() + i.mu.Lock() + defer i.mu.Unlock() + + if !i.dirty { + return nil + } snapshot := &cacheSnapshot{ Version: i.version, - Data: make(map[string]*Metadata), + Data: make(map[string]*Metadata, i.cache.EstimatedSize()), } - maps.Insert(snapshot.Data, i.cache.All()) // NOTE(dwisiswant0): write to temp for atomic op. @@ -229,11 +249,20 @@ func (i *Index) Save() error { return err } + i.dirty = false + return nil } // Load loads the cache from disk using gob decoding. func (i *Index) Load() error { + i.mu.Lock() + defer i.mu.Unlock() + + if i.dirty { + return errors.New("cannot load metadata cache into a modified index") + } + file, err := os.Open(i.cacheFile) if err != nil { if os.IsNotExist(err) { @@ -261,9 +290,6 @@ func (i *Index) Load() error { return nil } - i.mu.Lock() - defer i.mu.Unlock() - for key, value := range snapshot.Data { i.cache.Set(key, value) } @@ -296,11 +322,8 @@ func (i *Index) FilterFunc(fn FilterFunc) []string { return i.All() } - i.mu.RLock() - defer i.mu.RUnlock() - var matched []string - for path, metadata := range i.cache.All() { + for path, metadata := range i.GetAll() { if fn(metadata) { matched = append(matched, path) } @@ -327,7 +350,10 @@ func (i *Index) GetAll() map[string]*Metadata { i.mu.RLock() defer i.mu.RUnlock() - result := maps.Collect(i.cache.All()) + result := make(map[string]*Metadata, i.cache.EstimatedSize()) + for path, metadata := range i.cache.All() { + result[path] = metadata.clone() + } return result } diff --git a/pkg/catalog/index/index_test.go b/pkg/catalog/index/index_test.go index de76dd7592..c51a961adb 100644 --- a/pkg/catalog/index/index_test.go +++ b/pkg/catalog/index/index_test.go @@ -35,6 +35,13 @@ func TestNewIndex(t *testing.T) { }) } +func TestMetadataIsValidatedForRejectsUnknownMode(t *testing.T) { + metadata := &Metadata{Validation: ValidationMode(255)} + + require.False(t, metadata.IsValidatedFor(false)) + require.False(t, metadata.IsValidatedFor(true)) +} + func TestCacheBasicOperations(t *testing.T) { tmpDir := t.TempDir() cache, err := NewIndex(tmpDir) @@ -164,6 +171,143 @@ func TestCachePersistence(t *testing.T) { }) } +func TestCacheSaveSkipsUnchangedIndex(t *testing.T) { + tmpDir := t.TempDir() + cache, err := NewIndex(tmpDir) + require.NoError(t, err) + + cache.Set("/tmp/unchanged.yaml", &Metadata{ + ID: "unchanged", + FilePath: "/tmp/unchanged.yaml", + }) + require.NoError(t, cache.Save()) + + cacheFile := filepath.Join(tmpDir, IndexFileName) + oldTime := time.Unix(1, 0) + require.NoError(t, os.Chtimes(cacheFile, oldTime, oldTime)) + + require.NoError(t, cache.Save()) + + stat, err := os.Stat(cacheFile) + require.NoError(t, err) + require.Equal(t, oldTime, stat.ModTime(), "unchanged cache should not be rewritten") +} + +func TestCacheLoadRejectsModifiedIndex(t *testing.T) { + tmpDir := t.TempDir() + persisted, err := NewIndex(tmpDir) + require.NoError(t, err) + persisted.Set("/tmp/persisted.yaml", &Metadata{ID: "persisted", FilePath: "/tmp/persisted.yaml"}) + require.NoError(t, persisted.Save()) + + cache, err := NewIndex(tmpDir) + require.NoError(t, err) + cache.Set("/tmp/unsaved.yaml", &Metadata{ID: "unsaved", FilePath: "/tmp/unsaved.yaml"}) + + require.Error(t, cache.Load()) + require.True(t, cache.Has("/tmp/unsaved.yaml"), "rejected load must preserve unsaved entries") +} + +func TestCacheGetSynchronouslyInvalidatesStaleEntry(t *testing.T) { + tmpDir := t.TempDir() + templatePath := filepath.Join(tmpDir, "stale.yaml") + require.NoError(t, os.WriteFile(templatePath, []byte("id: stale"), 0o600)) + + cache, err := NewIndex(tmpDir) + require.NoError(t, err) + cache.Set(templatePath, &Metadata{ + ID: "stale", + FilePath: templatePath, + ModTime: time.Unix(1, 0), + }) + + metadata, found := cache.Get(templatePath) + require.False(t, found) + require.Nil(t, metadata) + require.False(t, cache.Has(templatePath), "Get must finish stale-entry invalidation before returning") +} + +func TestCacheMetadataIsDefensivelyCopied(t *testing.T) { + tmpDir := t.TempDir() + templatePath := filepath.Join(tmpDir, "metadata.yaml") + require.NoError(t, os.WriteFile(templatePath, []byte("id: metadata"), 0o600)) + info, err := os.Stat(templatePath) + require.NoError(t, err) + + cache, err := NewIndex(tmpDir) + require.NoError(t, err) + input := &Metadata{ + ID: "original", + FilePath: templatePath, + ModTime: info.ModTime(), + Authors: []string{"original-author"}, + Tags: []string{"original-tag"}, + } + cache.Set(templatePath, input) + + input.ID = "mutated-input" + input.Authors[0] = "mutated-input-author" + input.Tags[0] = "mutated-input-tag" + require.NoError(t, cache.Save()) + + fromGet, found := cache.Get(templatePath) + require.True(t, found) + fromGet.ID = "mutated-get" + fromGet.Authors[0] = "mutated-get-author" + + fromGetAll := cache.GetAll() + fromGetAll[templatePath].ID = "mutated-get-all" + fromGetAll[templatePath].Tags[0] = "mutated-get-all-tag" + require.NoError(t, cache.Save()) + + loaded, err := NewIndex(tmpDir) + require.NoError(t, err) + require.NoError(t, loaded.Load()) + metadata := loaded.GetAll()[templatePath] + require.Equal(t, "original", metadata.ID) + require.Equal(t, []string{"original-author"}, metadata.Authors) + require.Equal(t, []string{"original-tag"}, metadata.Tags) +} + +func TestCacheClearPersistsEmptyIndexWithoutLoad(t *testing.T) { + tmpDir := t.TempDir() + persisted, err := NewIndex(tmpDir) + require.NoError(t, err) + persisted.Set("/tmp/persisted.yaml", &Metadata{ID: "persisted", FilePath: "/tmp/persisted.yaml"}) + require.NoError(t, persisted.Save()) + + cache, err := NewIndex(tmpDir) + require.NoError(t, err) + cache.Clear() + require.NoError(t, cache.Save()) + + loaded, err := NewIndex(tmpDir) + require.NoError(t, err) + require.NoError(t, loaded.Load()) + require.Equal(t, 0, loaded.Size()) +} + +func TestFilterFuncCanMutateIndex(t *testing.T) { + cache, err := NewIndex(t.TempDir()) + require.NoError(t, err) + cache.Set("/tmp/original.yaml", &Metadata{ID: "original", FilePath: "/tmp/original.yaml"}) + + done := make(chan []string, 1) + go func() { + done <- cache.FilterFunc(func(*Metadata) bool { + cache.Set("/tmp/from-callback.yaml", &Metadata{ID: "callback", FilePath: "/tmp/from-callback.yaml"}) + return true + }) + }() + + select { + case paths := <-done: + require.Equal(t, []string{"/tmp/original.yaml"}, paths) + case <-time.After(time.Second): + t.Fatal("FilterFunc deadlocked when its callback mutated the index") + } +} + func TestIndexVersionMismatch(t *testing.T) { tmpDir := t.TempDir() @@ -182,7 +326,10 @@ func TestIndexVersionMismatch(t *testing.T) { require.NoError(t, err) // Manually modify version and save again + cache1.mu.Lock() cache1.version = 999 + cache1.dirty = true + cache1.mu.Unlock() err = cache1.Save() require.NoError(t, err) @@ -472,6 +619,19 @@ func TestCacheConcurrency(t *testing.T) { <-done } }) + + t.Run("Concurrent Save", func(t *testing.T) { + errs := make(chan error, 10) + for range 10 { + go func() { + errs <- cache.Save() + }() + } + + for range 10 { + require.NoError(t, <-errs) + } + }) } func TestCacheSize(t *testing.T) { diff --git a/pkg/catalog/index/metadata.go b/pkg/catalog/index/metadata.go index 013ab439c1..d1563a6479 100644 --- a/pkg/catalog/index/metadata.go +++ b/pkg/catalog/index/metadata.go @@ -10,6 +10,18 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" ) +// ValidationMode records the syntax policy used to validate cached metadata. +type ValidationMode uint8 + +const ( + // ValidationUnknown indicates that parser validation has not been recorded. + ValidationUnknown ValidationMode = iota + // ValidationLax indicates successful validation with unknown fields allowed. + ValidationLax + // ValidationStrict indicates successful strict syntax validation. + ValidationStrict +) + // Metadata contains lightweight metadata extracted from a template. type Metadata struct { // ID is the unique identifier of the template. @@ -42,6 +54,10 @@ type Metadata struct { // TemplateVerifier is the verifier used for the template. TemplateVerifier string `gob:"verifier,omitempty"` + // Validation records how the built-in parser validated this metadata's + // source before it was cached. + Validation ValidationMode `gob:"validation,omitempty"` + // NOTE(dwisiswant0): Consider adding more fields here in the future to // enhance filtering caps w/o loading full templates, such as: // `has_{code,headless,file}` to indicate presence of protocol-based @@ -54,6 +70,31 @@ type Metadata struct { // needed. } +func (m *Metadata) clone() *Metadata { + if m == nil { + return nil + } + + cloned := *m + cloned.Authors = slices.Clone(m.Authors) + cloned.Tags = slices.Clone(m.Tags) + + return &cloned +} + +// IsValidatedFor reports whether the cached validation satisfies the requested +// syntax policy. +func (m *Metadata) IsValidatedFor(strictSyntax bool) bool { + switch m.Validation { + case ValidationLax: + return !strictSyntax + case ValidationStrict: + return true + default: + return false + } +} + // NewMetadataFromTemplate creates a new metadata object from a template. func NewMetadataFromTemplate(path string, tpl *templates.Template) *Metadata { return &Metadata{ diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 5df72a7fe1..434a8f719f 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -72,6 +72,10 @@ type Config struct { Catalog catalog.Catalog ExecutorOptions *protocols.ExecutorOptions Logger *gologger.Logger + + // MetadataIndex is an optional shared index borrowed by the store. The + // caller remains responsible for persisting it. + MetadataIndex *index.Index } // Store is a storage for loaded nuclei templates @@ -170,14 +174,21 @@ func New(cfg *Config) (*Store, error) { return nil }) - // Initialize metadata index and filter (load from disk & cache for reuse) - store.metadataIndex = store.loadTemplatesIndex() + // Initialize metadata index and filter (load from disk & cache for reuse). + ownsMetadataIndex := cfg.MetadataIndex == nil + store.metadataIndex = cfg.MetadataIndex + if store.metadataIndex == nil { + store.metadataIndex = store.loadTemplatesIndex() + } store.indexFilter = store.buildIndexFilter() if cfg.ExecutorOptions != nil { - cfg.ExecutorOptions.TemplateVerificationCallback = store.getTemplateVerification + metadataIndex := store.metadataIndex + cfg.ExecutorOptions.TemplateVerificationCallback = func(templatePath string) *protocols.TemplateVerification { + return getTemplateVerification(metadataIndex, templatePath) + } } store.saveMetadataIndexOnce = sync.OnceFunc(func() { - if store.metadataIndex == nil { + if store.metadataIndex == nil || !ownsMetadataIndex { return } @@ -250,12 +261,12 @@ func New(cfg *Config) (*Store, error) { return store, nil } -func (store *Store) getTemplateVerification(templatePath string) *protocols.TemplateVerification { - if store.metadataIndex == nil { +func getTemplateVerification(metadataIndex *index.Index, templatePath string) *protocols.TemplateVerification { + if metadataIndex == nil { return nil } - metadata, found := store.metadataIndex.Get(templatePath) + metadata, found := metadataIndex.Get(templatePath) if !found { return nil } @@ -390,6 +401,31 @@ func (store *Store) loadTemplatesIndex() *index.Index { return metadataIdx } +func (store *Store) cacheValidatedMetadata(templatePath string, template *templates.Template) *index.Metadata { + metadata := index.NewMetadataFromTemplate(templatePath, template) + if parser, ok := store.config.ExecutorOptions.Parser.(*templates.Parser); ok { + metadata.Validation = index.ValidationStrict + if parser.NoStrictSyntax { + metadata.Validation = index.ValidationLax + } + } + + info, err := os.Stat(templatePath) + if err != nil { + return metadata + } + metadata.ModTime = info.ModTime() + + store.metadataIndex.Set(templatePath, metadata) + return metadata +} + +func (store *Store) metadataValidForParser(metadata *index.Metadata) bool { + parser, ok := store.config.ExecutorOptions.Parser.(*templates.Parser) + + return ok && metadata.IsValidatedFor(!parser.NoStrictSyntax) +} + // LoadTemplateTags loads template tags count using metadata index when possible. // // This method is optimized for tag listing (`-tgl`) and avoids loading all @@ -419,7 +455,7 @@ func (store *Store) LoadTemplateTags() (map[string]int, error) { continue } - if !requiresTemplateParse { + if !requiresTemplateParse && store.metadataValidForParser(metadata) { for _, tag := range metadata.Tags { tagsMap[tag]++ } @@ -453,7 +489,7 @@ func (store *Store) LoadTemplateTags() (map[string]int, error) { var metadata *index.Metadata if store.metadataIndex != nil { - metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, template) + metadata = store.cacheValidatedMetadata(templatePath, template) } else { metadata = index.NewMetadataFromTemplate(templatePath, template) } @@ -481,39 +517,46 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { validPaths := make(map[string]struct{}) for _, templatePath := range templatePaths { + var cachedMetadata *index.Metadata + if store.metadataIndex != nil { if metadata, found := store.metadataIndex.Get(templatePath); found { if !indexFilter.Matches(metadata) { continue } - if store.tagFilter != nil { - loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) - if !loaded { - if err != nil && strings.Contains(err.Error(), templates.ErrExcluded.Error()) { - stats.Increment(templates.ExcludedWeakMatcherTemplateStats) - if config.DefaultConfig.LogAllEvents { - store.logger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) - } - } - continue - } + cachedMetadata = metadata + } + } + + // Metadata-only loading still applies the configured tag filter. + loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) + if err != nil { + if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { + stats.Increment(templates.ExcludedWeakMatcherTemplateStats) + + if config.DefaultConfig.LogAllEvents { + store.logger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) } - validPaths[templatePath] = struct{}{} continue } + + store.logger.Warning().Msg(err.Error()) } - loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog) - if loaded { + if !loaded { + continue + } + + if cachedMetadata == nil { templatesCache := store.parserCacheOnce() if templatesCache != nil { if template, _, _ := templatesCache.Has(templatePath); template != nil { var metadata *index.Metadata if store.metadataIndex != nil { - metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, template) + metadata = store.cacheValidatedMetadata(templatePath, template) } else { metadata = index.NewMetadataFromTemplate(templatePath, template) } @@ -521,26 +564,11 @@ func (store *Store) LoadTemplatesOnlyMetadata() error { if !indexFilter.Matches(metadata) { continue } - - validPaths[templatePath] = struct{}{} - continue } } - - validPaths[templatePath] = struct{}{} } - if err != nil { - if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { - stats.Increment(templates.ExcludedWeakMatcherTemplateStats) - if config.DefaultConfig.LogAllEvents { - store.logger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) - } - continue - } - - store.logger.Warning().Msg(err.Error()) - } + validPaths[templatePath] = struct{}{} } templatesCache := store.parserCacheOnce() @@ -765,6 +793,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp defer store.saveMetadataIndexOnce() indexFilter := store.indexFilter + requiresTemplateParse := len(store.config.IncludeConditions) > 0 includedTemplates, errs := store.config.Catalog.GetTemplatesPath(templatesList) store.logErroredTemplates(errs) @@ -841,8 +870,8 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp defer wgLoadTemplates.Done() var ( - metadata *index.Metadata - metadataCached bool + metadata *index.Metadata + metadataReusable bool ) if store.metadataIndex != nil { @@ -852,21 +881,28 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) ([]*temp noteExcludedByTag(templatePath, metadata) return } - // NOTE(dwisiswant0): else, tagFilter probably exists (for - // IncludeConditions), which still need to check via - // LoadTemplate. - metadataCached = true + if len(tags) > 0 && !slices.ContainsFunc(tags, metadata.HasTag) { + return + } + + metadataReusable = store.metadataValidForParser(metadata) } } - loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) + var err error + + loaded := metadataReusable && !requiresTemplateParse + if !loaded { + loaded, err = store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) + } + if loaded { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) - if parsed != nil && !metadataCached { + if parsed != nil && !metadataReusable { if store.metadataIndex != nil { - metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, parsed) + metadata = store.cacheValidatedMetadata(templatePath, parsed) } else { metadata = index.NewMetadataFromTemplate(templatePath, parsed) } diff --git a/pkg/catalog/loader/loader_test.go b/pkg/catalog/loader/loader_test.go index 02ff6dd470..07718484a6 100644 --- a/pkg/catalog/loader/loader_test.go +++ b/pkg/catalog/loader/loader_test.go @@ -1,17 +1,23 @@ package loader import ( + "bytes" "os" "path/filepath" "reflect" "testing" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/formatter" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" + metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index" "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats" "github.com/stretchr/testify/require" ) @@ -53,6 +59,90 @@ func TestLoadTemplates(t *testing.T) { }) } +func TestNewUsesConfiguredMetadataIndex(t *testing.T) { + indexDir := t.TempDir() + metadataIndex, err := metadataindex.NewIndex(indexDir) + require.NoError(t, err) + + store, err := New(&Config{ + Catalog: disk.NewCatalog(""), + Logger: &gologger.Logger{}, + MetadataIndex: metadataIndex, + }) + require.NoError(t, err) + require.Same(t, metadataIndex, store.metadataIndex) + + metadataIndex.Set("/tmp/shared.yaml", &metadataindex.Metadata{ID: "shared"}) + store.saveMetadataIndexOnce() + _, err = os.Stat(filepath.Join(indexDir, metadataindex.IndexFileName)) + require.ErrorIs(t, err, os.ErrNotExist, "borrowed index persistence belongs to its owner") +} + +func TestCacheValidatedMetadataReturnsReplacement(t *testing.T) { + templatePath := filepath.Join(t.TempDir(), "replacement.yaml") + require.NoError(t, os.WriteFile(templatePath, []byte("id: replacement"), 0o600)) + info, err := os.Stat(templatePath) + require.NoError(t, err) + + metadataIndex, err := metadataindex.NewIndex(t.TempDir()) + require.NoError(t, err) + metadataIndex.Set(templatePath, &metadataindex.Metadata{ + ID: "old", + FilePath: templatePath, + ModTime: info.ModTime(), + }) + store := &Store{ + config: &Config{ExecutorOptions: &protocols.ExecutorOptions{Parser: templates.NewParser()}}, + metadataIndex: metadataIndex, + } + + metadata := store.cacheValidatedMetadata(templatePath, &templates.Template{ID: "replacement"}) + require.Equal(t, "replacement", metadata.ID) + cached, found := metadataIndex.Get(templatePath) + require.True(t, found) + require.Equal(t, "replacement", cached.ID) +} + +func TestLoadTemplatesOnlyMetadataLogsCachedTemplateParseErrors(t *testing.T) { + templatePath := filepath.Join(t.TempDir(), "invalid.yaml") + require.NoError(t, os.WriteFile(templatePath, []byte("id: invalid\ninfo: [\n"), 0o600)) + info, err := os.Stat(templatePath) + require.NoError(t, err) + + metadataIndex, err := metadataindex.NewIndex(t.TempDir()) + require.NoError(t, err) + metadataIndex.Set(templatePath, &metadataindex.Metadata{ + ID: "invalid", + FilePath: templatePath, + ModTime: info.ModTime(), + Validation: metadataindex.ValidationStrict, + }) + + var output bytes.Buffer + logger := &gologger.Logger{} + logger.SetFormatter(formatter.NewCLI(false)) + logger.SetWriter(&utils.CaptureWriter{Buffer: &output}) + logger.SetMaxLevel(levels.LevelDebug) + + options := testutils.DefaultOptions.Copy() + options.Logger = logger + options.Templates = []string{templatePath} + catalog := disk.NewCatalog("") + executerOpts := testutils.NewMockExecuterOptions(options, nil) + executerOpts.Catalog = catalog + executerOpts.Parser = templates.NewParser() + executerOpts.Logger = logger + + loaderConfig := NewConfig(options, catalog, executerOpts) + loaderConfig.MetadataIndex = metadataIndex + store, err := New(loaderConfig) + require.NoError(t, err) + + output.Reset() + require.NoError(t, store.LoadTemplatesOnlyMetadata()) + require.Contains(t, output.String(), "Could not load template") +} + func TestRemoteTemplates(t *testing.T) { catalog := disk.NewCatalog("") @@ -160,6 +250,147 @@ code: require.Equal(t, initialUnverified, stats.GetValue(templates.SkippedUnverifiedTemplateStats)) } +func TestLoadTemplatesSkipsMetadataParseForCachedIndexEntry(t *testing.T) { + templatePath := filepath.Join(t.TempDir(), "cached-index.yaml") + err := os.WriteFile(templatePath, []byte(`id: cached-index-template + +info: + name: Cached Index Template + author: pdteam + severity: info + tags: cached + +http: + - method: GET + path: + - "{{BaseURL}}" +`), 0o600) + require.NoError(t, err) + fileInfo, err := os.Stat(templatePath) + require.NoError(t, err) + + metadataIndex, err := metadataindex.NewIndex(t.TempDir()) + require.NoError(t, err) + metadataIndex.Set(templatePath, &metadataindex.Metadata{ + ID: "cached-index-template", + FilePath: templatePath, + ModTime: fileInfo.ModTime(), + Name: "Cached Index Template", + Authors: []string{"pdteam"}, + Tags: []string{"cached"}, + Severity: "info", + ProtocolType: "http", + Validation: metadataindex.ValidationStrict, + }) + + options := testutils.DefaultOptions.Copy() + options.Logger = &gologger.Logger{} + options.ExecutionId = "loader-cached-index-template" + options.DisableUnsignedTemplates = false + options.TemplateLoadingConcurrency = 1 + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + catalog := disk.NewCatalog("") + parser := templates.NewParser() + executerOpts := testutils.NewMockExecuterOptions(options, nil) + executerOpts.Catalog = catalog + executerOpts.Parser = parser + executerOpts.Logger = options.Logger + + workflowLoader, err := workflow.NewLoader(executerOpts) + require.NoError(t, err) + executerOpts.WorkflowLoader = workflowLoader + + loaderConfig := NewConfig(options, catalog, executerOpts) + loaderConfig.MetadataIndex = metadataIndex + store, err := New(loaderConfig) + require.NoError(t, err) + + loaded, err := store.LoadTemplates([]string{templatePath}) + require.NoError(t, err) + require.Len(t, loaded, 1) + require.Equal(t, 0, parser.ParsedCount(), "valid index metadata should avoid the filtering parse") + require.Equal(t, 1, parser.CompiledCount()) + + options.IncludeConditions = []string{`id == "does-not-match"`} + loaderConfig = NewConfig(options, catalog, executerOpts) + loaderConfig.MetadataIndex = metadataIndex + store, err = New(loaderConfig) + require.NoError(t, err) + + loaded, err = store.LoadTemplates([]string{templatePath}) + require.NoError(t, err) + require.Empty(t, loaded) + require.Equal(t, 1, parser.ParsedCount(), "advanced conditions still require the parsed template") +} + +func TestLoadTemplatesRevalidatesLaxMetadataInStrictMode(t *testing.T) { + templatePath := filepath.Join(t.TempDir(), "lax-metadata.yaml") + err := os.WriteFile(templatePath, []byte(`id: lax-metadata-template + +info: + name: Lax Metadata Template + author: pdteam + severity: info + +unknown-field: accepted-only-in-lax-mode + +http: + - method: GET + path: + - "{{BaseURL}}" +`), 0o600) + require.NoError(t, err) + + metadataIndex, err := metadataindex.NewIndex(t.TempDir()) + require.NoError(t, err) + catalog := disk.NewCatalog("") + + newStore := func(executionID string, noStrictSyntax bool) *Store { + options := testutils.DefaultOptions.Copy() + options.Logger = &gologger.Logger{} + options.ExecutionId = executionID + options.DisableUnsignedTemplates = false + options.TemplateLoadingConcurrency = 1 + options.NoStrictSyntax = noStrictSyntax + testutils.Init(options) + t.Cleanup(func() { + testutils.Cleanup(options) + }) + + parser := templates.NewParser() + parser.NoStrictSyntax = noStrictSyntax + executerOpts := testutils.NewMockExecuterOptions(options, nil) + executerOpts.Catalog = catalog + executerOpts.Parser = parser + executerOpts.Logger = options.Logger + + workflowLoader, err := workflow.NewLoader(executerOpts) + require.NoError(t, err) + executerOpts.WorkflowLoader = workflowLoader + + loaderConfig := NewConfig(options, catalog, executerOpts) + loaderConfig.MetadataIndex = metadataIndex + store, err := New(loaderConfig) + require.NoError(t, err) + return store + } + + laxStore := newStore("loader-lax-metadata", true) + loaded, err := laxStore.LoadTemplates([]string{templatePath}) + require.NoError(t, err) + require.Len(t, loaded, 1) + require.True(t, metadataIndex.Has(templatePath)) + + strictStore := newStore("loader-strict-metadata", false) + loaded, err = strictStore.LoadTemplates([]string{templatePath}) + require.NoError(t, err) + require.Empty(t, loaded, "strict loading must not trust metadata validated only in lax mode") +} + func loadSingleTemplateForTest(t *testing.T, templatePath, executionID string) []*templates.Template { t.Helper()