perf(loader): reuse cached parsed templates

pkg/catalog/loader/loader.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
@@ -88,6 +89,9 @@ type Store struct {
 
 	logger *gologger.Logger
 
+	// parserCacheOnce is used to cache the parser cache result
+	parserCacheOnce func() *templates.Cache
+
 	// NotFoundCallback is called for each not found template
 	// This overrides error handling for not found templates
 	NotFoundCallback func(template string) bool
@@ -155,6 +159,18 @@ func New(cfg *Config) (*Store, error) {
 		logger:         cfg.Logger,
 	}
 
+	store.parserCacheOnce = sync.OnceValue(func() *templates.Cache {
+		if cfg.ExecutorOptions == nil || cfg.ExecutorOptions.Parser == nil {
+			return nil
+		}
+
+		if parser, ok := cfg.ExecutorOptions.Parser.(*templates.Parser); ok {
+			return parser.Cache()
+		}
+
+		return nil
+	})
+
 	// Do a check to see if we have URLs in templates flag, if so
 	// we need to processs them separately and remove them from the initial list
 	var templatesFinal []string
@@ -310,11 +326,11 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
 			store.logger.Warning().Msg(err.Error())
 		}
 	}
-	parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser)
-	if !ok {
+
+	templatesCache := store.parserCacheOnce()
+	if templatesCache == nil {
 		return errors.New("invalid parser")
 	}
-	templatesCache := parserItem.Cache()
 
 	loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]()
 
@@ -386,6 +402,7 @@ func (store *Store) areTemplatesValid(filteredTemplatePaths map[string]struct{})
 
 func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *templates.TagFilter) (bool, error)) bool {
 	areTemplatesValid := true
+	parsedCache := store.parserCacheOnce()
 
 	for templatePath := range filteredTemplatePaths {
 		if _, err := load(templatePath, store.tagFilter); err != nil {
@@ -395,13 +412,26 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
 			}
 		}
 
-		template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
-		if err != nil {
-			if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
-				areTemplatesValid = false
-				continue
+		var template *templates.Template
+		var err error
+
+		if parsedCache != nil {
+			if cachedTemplate, _, cacheErr := parsedCache.Has(templatePath); cacheErr == nil && cachedTemplate != nil {
+				template = cachedTemplate
+			}
+		}
+
+		if template == nil {
+			template, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
+			if err != nil {
+				if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
+					areTemplatesValid = false
+					continue
+				}
 			}
-		} else if template == nil {
+		}
+
+		if template == nil {
 			// NOTE(dwisiswant0): possibly global matchers template.
 			// This could definitely be handled better, for example by returning an
 			// `ErrGlobalMatchersTemplate` during `templates.Parse` and checking it

pkg/catalog/loader/loader_bench_test.go

@@ -0,0 +1,43 @@
+package loader_test
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+)
+
+func BenchmarkStoreValidateTemplates(b *testing.B) {
+	options := testutils.DefaultOptions.Copy()
+	options.Logger = &gologger.Logger{}
+	testutils.Init(options)
+
+	catalog := disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	executerOpts := testutils.NewMockExecuterOptions(options, nil)
+	executerOpts.Parser = templates.NewParser()
+
+	workflowLoader, err := workflow.NewLoader(executerOpts)
+	if err != nil {
+		b.Fatalf("could not create workflow loader: %s", err)
+	}
+	executerOpts.WorkflowLoader = workflowLoader
+
+	loaderCfg := loader.NewConfig(options, catalog, executerOpts)
+
+	store, err := loader.New(loaderCfg)
+	if err != nil {
+		b.Fatalf("could not create store: %s", err)
+	}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for b.Loop() {
+		_ = store.ValidateTemplates()
+	}
+}