fix(loader): compile requests during validation

pkg/catalog/loader/loader.go

@@ -656,7 +656,6 @@ func (store *Store) areTemplatesValid(filteredTemplatePaths map[string]struct{})
 
 func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *templates.TagFilter) (bool, error)) bool {
 	areTemplatesValid := true
-	parsedCache := store.parserCacheOnce()
 
 	for templatePath := range filteredTemplatePaths {
 		if _, err := load(templatePath, store.tagFilter); err != nil {
@@ -666,22 +665,14 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
 			}
 		}
 
-		var template *templates.Template
-		var err error
-
-		if parsedCache != nil {
-			if cachedTemplate, _, cacheErr := parsedCache.Has(templatePath); cacheErr == nil && cachedTemplate != nil {
-				template = cachedTemplate
-			}
-		}
-
-		if template == nil {
-			template, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
-			if err != nil {
-				if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
-					areTemplatesValid = false
-					continue
-				}
+		// The load step validates the parsed definition and filters templates.
+		// FYI parse must still run because protocol compilation can surface
+		// additional validation errors.
+		template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
+		if err != nil {
+			if isParsingError(store, "Error occurred parsing template %s: %s\n", templatePath, err) {
+				areTemplatesValid = false
+				continue
 			}
 		}
 

pkg/catalog/loader/loader_test.go

@@ -143,6 +143,60 @@ func TestLoadTemplatesOnlyMetadataLogsCachedTemplateParseErrors(t *testing.T) {
 	require.Contains(t, output.String(), "Could not load template")
 }
 
+func TestValidateTemplatesRejectsRuntimeCompilationError(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "invalid-regex-group.yaml")
+	require.NoError(t, os.WriteFile(templatePath, []byte(`id: invalid-regex-group
+
+info:
+  name: Invalid Regex Group
+  author: pdteam
+  severity: info
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    extractors:
+      - type: regex
+        group: -1
+        regex:
+          - "(a)(b)"
+`), 0o600))
+
+	options := testutils.DefaultOptions.Copy()
+	var output bytes.Buffer
+	logger := &gologger.Logger{}
+	logger.SetFormatter(formatter.NewCLI(false))
+	logger.SetWriter(&utils.CaptureWriter{Buffer: &output})
+	logger.SetMaxLevel(levels.LevelDebug)
+	options.Logger = logger
+	options.ExecutionId = "loader-invalid-regex-group"
+	options.Templates = []string{templatePath}
+	options.Validate = true
+	options.TemplateLoadingConcurrency = 1
+	testutils.Init(options)
+	t.Cleanup(func() {
+		testutils.Cleanup(options)
+	})
+
+	catalog := disk.NewCatalog("")
+	executerOpts := testutils.NewMockExecuterOptions(options, nil)
+	executerOpts.Catalog = catalog
+	parser := templates.NewParser()
+	parser.ShouldValidate = true
+	executerOpts.Parser = parser
+	executerOpts.Logger = options.Logger
+
+	workflowLoader, err := workflow.NewLoader(executerOpts)
+	require.NoError(t, err)
+	executerOpts.WorkflowLoader = workflowLoader
+
+	store, err := New(NewConfig(options, catalog, executerOpts))
+	require.NoError(t, err)
+	require.Error(t, store.ValidateTemplates())
+	require.Contains(t, output.String(), "regex extractor group must be >= 0")
+}
+
 func TestRemoteTemplates(t *testing.T) {
 	catalog := disk.NewCatalog("")