diff --git a/internal/runner/preflight_portscan.go b/internal/runner/preflight_portscan.go index 26691882ef..3ebef3d89c 100644 --- a/internal/runner/preflight_portscan.go +++ b/internal/runner/preflight_portscan.go @@ -54,7 +54,7 @@ func (f *filteringInputProvider) SetWithExclusions(executionId string, value str } func (f *filteringInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { f.base.Iterate(func(mi *contextargs.MetaInput) bool { - key, err := mi.MarshalString() + key, err := preflightInputKey(mi) if err != nil { return callback(mi) } @@ -65,6 +65,29 @@ func (f *filteringInputProvider) Iterate(callback func(value *contextargs.MetaIn }) } +func preflightInputKey(metaInput *contextargs.MetaInput) (string, error) { + if metaInput.TargetFilter == nil { + // Preserve the exact historical key for all non-JSONL/legacy inputs. + return metaInput.MarshalString() + } + + // Marshal the legacy fields without TargetFilter so filter list ordering + // cannot affect the key. MetaInput.ID() contributes TargetFilter's + // canonical, presence-aware identity when used on a filter-only input. + legacyInput := contextargs.NewMetaInput() + legacyInput.Input = metaInput.Input + legacyInput.CustomIP = metaInput.CustomIP + legacyInput.ReqResp = metaInput.ReqResp + legacyKey, err := legacyInput.MarshalString() + if err != nil { + return "", err + } + + filterInput := contextargs.NewMetaInput() + filterInput.TargetFilter = metaInput.TargetFilter + return legacyKey + "\x00target-filter:" + filterInput.ID(), nil +} + // preflightResolveAndPortScan resolves hostname targets and performs a TCP connect scan for ports // required by loaded templates. Targets that are non-resolvable hostnames or have no relevant open // ports are filtered out from the input provider. @@ -97,7 +120,7 @@ func (r *Runner) preflightResolveAndPortScan(store *loader.Store) error { var totalTargets atomic.Int64 r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool { totalTargets.Add(1) - key, err := mi.MarshalString() + key, err := preflightInputKey(mi) if err != nil { return true } @@ -282,20 +305,20 @@ func (r *Runner) preflightResolveAndPortScan(store *loader.Store) error { close(stopProgress) // Apply filtering wrapper - allowedAll := allowed.GetAll() + allowedInputCount := countAllowedPreflightInputs(inputs, allowed) r.inputProvider = &filteringInputProvider{ base: r.inputProvider, allowed: allowed, - allowCnt: int64(len(allowedAll)), + allowCnt: allowedInputCount, execID: r.options.ExecutionId, } // Summary if !r.options.Silent { - dropped := totalTargets.Load() - kept.Load() + dropped := totalTargets.Load() - allowedInputCount r.Logger.Info().Msgf("Preflight summary: total=%d kept=%d filtered_dns=%d filtered_ports=%d", - totalTargets.Load(), kept.Load(), dnsFail.Load(), portFail.Load()) - r.Logger.Info().Msgf("Preflight targets: dropped=%d left=%d", dropped, kept.Load()) + totalTargets.Load(), allowedInputCount, dnsFail.Load(), portFail.Load()) + r.Logger.Info().Msgf("Preflight targets: dropped=%d left=%d", dropped, allowedInputCount) perPortOpenAll := perPortOpen.GetAll() if len(perPortOpenAll) > 0 { type kv struct { @@ -327,6 +350,16 @@ func (r *Runner) preflightResolveAndPortScan(store *loader.Store) error { return nil } +func countAllowedPreflightInputs(inputs []preflightTarget, allowed *mapsutil.SyncLockMap[string, struct{}]) int64 { + var count int64 + for _, input := range inputs { + if _, ok := allowed.Get(input.key); ok { + count++ + } + } + return count +} + type preflightTarget struct { key string target string diff --git a/internal/runner/preflight_portscan_test.go b/internal/runner/preflight_portscan_test.go new file mode 100644 index 0000000000..17b3d5db79 --- /dev/null +++ b/internal/runner/preflight_portscan_test.go @@ -0,0 +1,143 @@ +package runner + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" + inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + mapsutil "github.com/projectdiscovery/utils/maps" + "github.com/stretchr/testify/require" +) + +type preflightInputProviderStub struct { + inputs []*contextargs.MetaInput +} + +func (p *preflightInputProviderStub) Count() int64 { + return int64(len(p.inputs)) +} + +func (p *preflightInputProviderStub) Iterate(callback func(*contextargs.MetaInput) bool) { + for _, input := range p.inputs { + if !callback(input) { + return + } + } +} + +func (*preflightInputProviderStub) Set(string, string) {} +func (*preflightInputProviderStub) SetWithProbe(string, string, inputtypes.InputLivenessProbe) error { + return nil +} +func (*preflightInputProviderStub) SetWithExclusions(string, string) error { return nil } +func (*preflightInputProviderStub) InputType() string { return provider.TargetInputProvider } +func (*preflightInputProviderStub) Close() {} + +func TestPreflightInputKeyPreservesLegacyMarshalString(t *testing.T) { + input := contextargs.NewMetaInput() + input.Input = "https://example.com" + input.CustomIP = "192.0.2.1" + + marshaled, err := input.MarshalString() + require.NoError(t, err) + + key, err := preflightInputKey(input) + require.NoError(t, err) + require.Equal(t, marshaled, key) +} + +func TestPreflightInputKeySeparatesPresenceAwareFilters(t *testing.T) { + omitted := newPreflightFilteredInput(&contextargs.TargetFilter{}) + explicitEmpty := newPreflightFilteredInput(&contextargs.TargetFilter{ + HasTags: true, + Tags: []string{}, + }) + + // TargetFilter's Has* fields are intentionally not serialized, so the old + // MarshalString key could not distinguish these two JSONL records. + omittedMarshaled, err := omitted.MarshalString() + require.NoError(t, err) + explicitEmptyMarshaled, err := explicitEmpty.MarshalString() + require.NoError(t, err) + require.Equal(t, omittedMarshaled, explicitEmptyMarshaled) + + omittedKey, err := preflightInputKey(omitted) + require.NoError(t, err) + explicitEmptyKey, err := preflightInputKey(explicitEmpty) + require.NoError(t, err) + require.NotEqual(t, omittedKey, explicitEmptyKey) + + // Semantically equivalent filter lists produce one stable canonical key, + // independent of input ordering. + firstOrdering := newPreflightFilteredInput(&contextargs.TargetFilter{ + HasTags: true, + Tags: []string{"beta", "alpha"}, + }) + secondOrdering := newPreflightFilteredInput(&contextargs.TargetFilter{ + HasTags: true, + Tags: []string{"alpha", "beta"}, + }) + firstKey, err := preflightInputKey(firstOrdering) + require.NoError(t, err) + secondKey, err := preflightInputKey(secondOrdering) + require.NoError(t, err) + require.Equal(t, firstKey, secondKey) + + allowed := mapsutil.NewSyncLockMap[string, struct{}]() + require.NoError(t, allowed.Set(omittedKey, struct{}{})) + require.NoError(t, allowed.Set(explicitEmptyKey, struct{}{})) + + filtered := &filteringInputProvider{ + base: &preflightInputProviderStub{inputs: []*contextargs.MetaInput{omitted, explicitEmpty}}, + allowed: allowed, + allowCnt: 2, + execID: "test", + } + var iterated []*contextargs.MetaInput + filtered.Iterate(func(input *contextargs.MetaInput) bool { + iterated = append(iterated, input) + return true + }) + require.EqualValues(t, 2, filtered.Count()) + require.Len(t, iterated, 2) +} + +func TestFilteringInputProviderCountsKeptInputsNotAllowedKeys(t *testing.T) { + first := contextargs.NewMetaInput() + first.Input = "https://duplicate.example" + second := contextargs.NewMetaInput() + second.Input = first.Input + + key, err := preflightInputKey(first) + require.NoError(t, err) + allowed := mapsutil.NewSyncLockMap[string, struct{}]() + require.NoError(t, allowed.Set(key, struct{}{})) + require.Len(t, allowed.GetAll(), 1) + require.EqualValues(t, 2, countAllowedPreflightInputs([]preflightTarget{ + {key: key}, + {key: key}, + }, allowed)) + + filtered := &filteringInputProvider{ + base: &preflightInputProviderStub{inputs: []*contextargs.MetaInput{first, second}}, + allowed: allowed, + allowCnt: 2, + execID: "test", + } + var iterated int + filtered.Iterate(func(*contextargs.MetaInput) bool { + iterated++ + return true + }) + + require.EqualValues(t, 2, filtered.Count()) + require.Equal(t, 2, iterated) +} + +func newPreflightFilteredInput(filter *contextargs.TargetFilter) *contextargs.MetaInput { + input := contextargs.NewMetaInput() + input.Input = "https://duplicate.example" + input.TargetFilter = filter + return input +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 98a87e1bf0..73d0a31dcd 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -680,7 +680,10 @@ func (r *Runner) RunEnumeration() error { // If using input-file flags, only load http fuzzing based templates. loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts) - if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST { + if err := r.prepareTargetFilters(loaderConfig); err != nil { + return errors.Wrap(err, "could not prepare per-target JSONL filters") + } + if r.inputProvider.InputType() == provider.MultiFormatInputProvider || r.options.DAST { // if input type is not list (implicitly enable fuzzing) r.options.DAST = true } diff --git a/internal/runner/target_filters.go b/internal/runner/target_filters.go new file mode 100644 index 0000000000..243c736a4c --- /dev/null +++ b/internal/runner/target_filters.go @@ -0,0 +1,310 @@ +package runner + +import ( + "fmt" + "net/url" + "path/filepath" + "sort" + "strings" + + "github.com/projectdiscovery/nuclei/v3/pkg/catalog" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" +) + +// prepareTargetFilters resolves JSONL inheritance before template loading. The +// loader is widened to the union of all per-target selections, while each +// MetaInput keeps an effective filter that narrows execution back to that +// target's criteria. +func (r *Runner) prepareTargetFilters(loaderConfig *loader.Config) error { + targetProvider, ok := r.inputProvider.(provider.PerTargetInputProvider) + if !ok || r.inputProvider.InputType() != provider.TargetInputProvider { + return nil + } + + inputs := targetProvider.TargetInputs() + if len(inputs) == 0 { + return fmt.Errorf("jsonl target input did not contain any targets") + } + + hasOverrides := false + hasTemplateOverrides := false + for _, input := range inputs { + filter := input.TargetFilter + if filter == nil { + return fmt.Errorf("jsonl target %q is missing filter metadata", input.Input) + } + if filter.HasTags || filter.HasExcludeTags || filter.HasSeverities || filter.HasTemplates { + hasOverrides = true + } + if filter.HasTemplates { + hasTemplateOverrides = true + } + } + if !hasOverrides { + // With no per-target overrides, leave the loader's normal CLI behavior + // untouched and remove empty metadata so IDs/hashes remain compatible. + for _, input := range inputs { + input.TargetFilter = nil + } + return nil + } + if r.options.AutomaticScan { + return fmt.Errorf("automatic scan is not supported with per-target JSONL overrides") + } + if len(r.options.Workflows) > 0 || len(r.options.WorkflowURLs) > 0 { + return fmt.Errorf("workflows are not supported with per-target JSONL overrides") + } + if r.options.EnableGlobalMatchersTemplates { + return fmt.Errorf("global matchers are not supported with per-target JSONL overrides") + } + if hasTemplateOverrides && len(r.options.TemplateURLs) > 0 { + return fmt.Errorf("global remote template URLs are not supported together with per-target template overrides") + } + + // Per-target tag/severity combinations cannot be represented by the + // loader's global AND filter. Load a safe superset, while retaining the + // built-in ignore list and all unrelated global filters. + loaderConfig.Tags = nil + loaderConfig.Severities = nil + loaderConfig.ExcludeTags = append([]string(nil), config.ReadIgnoreFile().Tags...) + + var ( + unionTemplatePaths = make(map[string]struct{}) + loadDefaultTemplates bool + ) + + var globalTemplatePaths []string + if hasTemplateOverrides && len(r.options.Templates) > 0 { + resolved, err := resolveTargetTemplatePaths(r.catalog, r.options.Templates, 0, false) + if err != nil { + return fmt.Errorf("could not resolve global templates for JSONL target inheritance: %w", err) + } + globalTemplatePaths = resolved + for _, path := range resolved { + unionTemplatePaths[path] = struct{}{} + } + } + + var globalIncludeTemplatePaths []string + if len(r.options.IncludeTemplates) > 0 { + resolved, err := resolveTargetTemplatePaths(r.catalog, r.options.IncludeTemplates, 0, false) + if err != nil { + return fmt.Errorf("could not resolve globally included templates for JSONL target inheritance: %w", err) + } + globalIncludeTemplatePaths = resolved + } + + var defaultTemplatePaths []string + if hasTemplateOverrides { + for _, input := range inputs { + filter := input.TargetFilter + if (filter.HasTemplates && len(filter.Templates) == 0) || + (!filter.HasTemplates && len(r.options.Templates) == 0) { + loadDefaultTemplates = true + break + } + } + if loadDefaultTemplates { + resolved, err := resolveTargetTemplatePaths(r.catalog, []string{config.DefaultConfig.TemplatesDirectory}, 0, false) + if err != nil { + return fmt.Errorf("could not resolve default templates for JSONL target inheritance: %w", err) + } + defaultTemplatePaths = resolved + } + } + + resolvedTemplateOverrides := make(map[string][]string) + for _, input := range inputs { + filter := input.TargetFilter + + effectiveTags := []string(r.options.Tags) + if filter.HasTags { + effectiveTags = filter.Tags + } + + effectiveExcludeTags := []string(r.options.ExcludeTags) + if filter.HasExcludeTags { + effectiveExcludeTags = filter.ExcludeTags + } + effectiveExcludeTags = removeIncludedTags(effectiveExcludeTags, []string(r.options.IncludeTags)) + + effectiveSeverities := append(severity.Severities(nil), r.options.Severities...) + if filter.HasSeverities { + effectiveSeverities = append(severity.Severities(nil), filter.Severities...) + } + + var ( + effectiveTemplatePaths []string + restrictTemplates bool + ) + if hasTemplateOverrides { + switch { + case filter.HasTemplates && len(filter.Templates) == 0: + // An explicitly empty -templates equivalent clears the global + // selection and falls back to the default template catalog. + effectiveTemplatePaths = defaultTemplatePaths + restrictTemplates = true + case filter.HasTemplates: + cacheKey := canonicalTargetSelectors(filter.Templates) + resolved, ok := resolvedTemplateOverrides[cacheKey] + if !ok { + var err error + resolved, err = resolveTargetTemplatePaths(r.catalog, filter.Templates, filter.SourceLine, true) + if err != nil { + return err + } + resolvedTemplateOverrides[cacheKey] = resolved + } + effectiveTemplatePaths = resolved + restrictTemplates = true + for _, path := range resolved { + unionTemplatePaths[path] = struct{}{} + } + case len(r.options.Templates) > 0: + effectiveTemplatePaths = globalTemplatePaths + restrictTemplates = true + default: + effectiveTemplatePaths = defaultTemplatePaths + restrictTemplates = true + } + } + + filter.Prepare( + append([]string(nil), effectiveTags...), + append([]string(nil), effectiveExcludeTags...), + effectiveSeverities, + effectiveTemplatePaths, + globalIncludeTemplatePaths, + restrictTemplates, + ) + } + + if hasTemplateOverrides { + templates := sortedKeys(unionTemplatePaths) + if loadDefaultTemplates { + templates = append([]string{config.DefaultConfig.TemplatesDirectory}, templates...) + } + loaderConfig.Templates = templates + } + return nil +} + +// resolveTargetTemplatePaths resolves template selectors to on-disk paths. +// When enforceLocalContainment is set (per-target JSONL selectors), a selector +// may not escape the templates tree via an absolute path or parent-directory +// traversal. Global -t/-it selectors are trusted CLI input and are resolved +// without that restriction so their documented arbitrary-path behavior is kept. +func resolveTargetTemplatePaths(templateCatalog catalog.Catalog, selectors []string, sourceLine int, enforceLocalContainment bool) ([]string, error) { + for _, selector := range selectors { + parsed, err := url.Parse(selector) + if err == nil && parsed.IsAbs() && (parsed.Scheme == "http" || parsed.Scheme == "https") { + location := "global options" + if sourceLine > 0 { + location = fmt.Sprintf("jsonl line %d", sourceLine) + } + return nil, fmt.Errorf("%s: remote template selector %q is not supported with per-target template overrides", location, selector) + } + if enforceLocalContainment && selectorEscapesTemplatesTree(selector) { + location := "template selector" + if sourceLine > 0 { + location = fmt.Sprintf("jsonl line %d template selector", sourceLine) + } + return nil, fmt.Errorf("%s %q must stay within the templates directory; absolute paths and parent-directory traversal are not allowed", location, selector) + } + } + + paths, pathErrors := templateCatalog.GetTemplatesPath(selectors) + if len(pathErrors) > 0 { + keys := make([]string, 0, len(pathErrors)) + for selector := range pathErrors { + keys = append(keys, selector) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, selector := range keys { + parts = append(parts, fmt.Sprintf("%s: %v", selector, pathErrors[selector])) + } + prefix := "template selector" + if sourceLine > 0 { + prefix = fmt.Sprintf("jsonl line %d template selector", sourceLine) + } + return nil, fmt.Errorf("%s could not be resolved: %s", prefix, strings.Join(parts, "; ")) + } + if len(paths) == 0 { + prefix := "template selectors" + if sourceLine > 0 { + prefix = fmt.Sprintf("jsonl line %d templates", sourceLine) + } + return nil, fmt.Errorf("%s did not match any templates", prefix) + } + + unique := make(map[string]struct{}, len(paths)) + for _, path := range paths { + unique[filepath.Clean(path)] = struct{}{} + } + return sortedKeys(unique), nil +} + +// selectorEscapesTemplatesTree reports whether a per-target template selector +// would resolve outside the templates directory. Both vectors are rejected at +// the input boundary, before any filesystem resolution touches the path: an +// absolute path (returned as-is by DiskCatalog.ResolvePath) and a relative path +// whose cleaned form climbs out of the tree via "..". +func selectorEscapesTemplatesTree(selector string) bool { + if filepath.IsAbs(selector) { + return true + } + cleaned := filepath.ToSlash(filepath.Clean(selector)) + return cleaned == ".." || strings.HasPrefix(cleaned, "../") +} + +func canonicalTargetSelectors(selectors []string) string { + values := append([]string(nil), selectors...) + sort.Strings(values) + return strings.Join(values, "\x00") +} + +func removeIncludedTags(excluded, included []string) []string { + includedSet := make(map[string]struct{}, len(included)) + for _, tag := range included { + includedSet[strings.ToLower(strings.TrimSpace(tag))] = struct{}{} + } + + result := make([]string, 0, len(excluded)) + seen := make(map[string]struct{}, len(excluded)) + for _, tag := range excluded { + tag = strings.ToLower(strings.TrimSpace(tag)) + if tag == "" { + continue + } + if _, allowed := includedSet[tag]; allowed { + continue + } + if _, ok := seen[tag]; ok { + continue + } + seen[tag] = struct{}{} + result = append(result, tag) + } + return result +} + +func sortedKeys(values map[string]struct{}) []string { + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Strings(result) + return result +} + +func targetFilterFor(input *contextargs.MetaInput) *contextargs.TargetFilter { + if input == nil { + return nil + } + return input.TargetFilter +} diff --git a/internal/runner/target_filters_test.go b/internal/runner/target_filters_test.go new file mode 100644 index 0000000000..2c6bc82fa9 --- /dev/null +++ b/internal/runner/target_filters_test.go @@ -0,0 +1,318 @@ +package runner + +import ( + "fmt" + "io" + "strings" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" + inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/stretchr/testify/require" +) + +type targetFilterTestProvider struct { + inputs []*contextargs.MetaInput +} + +func (p *targetFilterTestProvider) Count() int64 { + return int64(len(p.inputs)) +} + +func (p *targetFilterTestProvider) Iterate(callback func(*contextargs.MetaInput) bool) { + for _, input := range p.inputs { + if !callback(input.Clone()) { + return + } + } +} + +func (*targetFilterTestProvider) Set(string, string) {} +func (*targetFilterTestProvider) SetWithProbe(string, string, inputtypes.InputLivenessProbe) error { + return nil +} +func (*targetFilterTestProvider) SetWithExclusions(string, string) error { return nil } +func (*targetFilterTestProvider) InputType() string { return "TargetInputProvider" } +func (*targetFilterTestProvider) Close() {} +func (p *targetFilterTestProvider) TargetInputs() []*contextargs.MetaInput { + return p.inputs +} + +type targetFilterTestCatalog struct{} + +func (targetFilterTestCatalog) OpenFile(string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil +} + +func (targetFilterTestCatalog) GetTemplatePath(target string) ([]string, error) { + if target == "" { + return nil, fmt.Errorf("empty selector") + } + return []string{"/resolved/" + strings.TrimPrefix(target, "/")}, nil +} + +func (catalog targetFilterTestCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) { + var paths []string + errorsBySelector := make(map[string]error) + for _, definition := range definitions { + resolved, err := catalog.GetTemplatePath(definition) + if err != nil { + errorsBySelector[definition] = err + continue + } + paths = append(paths, resolved...) + } + return paths, errorsBySelector +} + +func (targetFilterTestCatalog) ResolvePath(templateName, _ string) (string, error) { + return "/resolved/" + strings.TrimPrefix(templateName, "/"), nil +} + +type countingTargetFilterTestCatalog struct { + targetFilterTestCatalog + calls map[string]int +} + +func (c *countingTargetFilterTestCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) { + c.calls[canonicalTargetSelectors(definitions)]++ + return c.targetFilterTestCatalog.GetTemplatesPath(definitions) +} + +func TestPrepareTargetFiltersKeepsDefaultAndCustomTemplateUnion(t *testing.T) { + inputs := []*contextargs.MetaInput{ + { + Input: "https://default.example", + TargetFilter: &contextargs.TargetFilter{ + HasTemplates: true, + Templates: []string{}, + SourceLine: 1, + }, + }, + { + Input: "https://custom.example", + TargetFilter: &contextargs.TargetFilter{ + HasTemplates: true, + Templates: []string{"custom.yaml"}, + SourceLine: 2, + }, + }, + } + runner := &Runner{ + options: &types.Options{}, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: inputs}, + } + loaderConfig := &loader.Config{} + + require.NoError(t, runner.prepareTargetFilters(loaderConfig)) + require.Contains(t, loaderConfig.Templates, config.DefaultConfig.TemplatesDirectory) + require.Contains(t, loaderConfig.Templates, "/resolved/custom.yaml") + + defaultTemplatePath := "/resolved/" + strings.TrimPrefix(config.DefaultConfig.TemplatesDirectory, "/") + require.True(t, inputs[0].TargetFilter.MatchesTemplate(defaultTemplatePath, nil, severity.Undefined, false)) + require.False(t, inputs[0].TargetFilter.MatchesTemplate("/resolved/custom.yaml", nil, severity.Undefined, false)) + require.False(t, inputs[1].TargetFilter.MatchesTemplate(defaultTemplatePath, nil, severity.Undefined, false)) + require.True(t, inputs[1].TargetFilter.MatchesTemplate("/resolved/custom.yaml", nil, severity.Undefined, false)) +} + +func TestPrepareTargetFiltersCachesSharedTemplateSelectors(t *testing.T) { + catalog := &countingTargetFilterTestCatalog{calls: make(map[string]int)} + inputs := []*contextargs.MetaInput{ + { + Input: "https://one.example", + TargetFilter: &contextargs.TargetFilter{ + HasTemplates: true, + Templates: []string{"shared.yaml"}, + SourceLine: 1, + }, + }, + { + Input: "https://two.example", + TargetFilter: &contextargs.TargetFilter{ + HasTemplates: true, + Templates: []string{"shared.yaml"}, + SourceLine: 2, + }, + }, + } + runner := &Runner{ + options: &types.Options{}, + catalog: catalog, + inputProvider: &targetFilterTestProvider{inputs: inputs}, + } + + require.NoError(t, runner.prepareTargetFilters(&loader.Config{})) + require.Equal(t, 1, catalog.calls[canonicalTargetSelectors([]string{"shared.yaml"})]) + require.True(t, inputs[0].TargetFilter.MatchesTemplate("/resolved/shared.yaml", nil, severity.Undefined, false)) + require.True(t, inputs[1].TargetFilter.MatchesTemplate("/resolved/shared.yaml", nil, severity.Undefined, false)) +} + +func TestPrepareTargetFiltersPreservesInheritanceAndForcedIncludes(t *testing.T) { + filter := &contextargs.TargetFilter{} + options := &types.Options{ + Severities: severity.Severities{severity.High}, + } + options.Tags = append(options.Tags, "apache") + options.ExcludeTags = append(options.ExcludeTags, "deprecated") + options.IncludeTemplates = append(options.IncludeTemplates, "forced.yaml") + + runner := &Runner{ + options: options, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + {Input: "https://example.com", TargetFilter: filter}, + { + Input: "https://override.example.com", + TargetFilter: &contextargs.TargetFilter{ + HasTags: true, + Tags: []string{"nginx"}, + }, + }, + }}, + } + loaderConfig := &loader.Config{} + + require.NoError(t, runner.prepareTargetFilters(loaderConfig)) + require.True(t, filter.MatchesTemplate("/resolved/normal.yaml", []string{"apache"}, severity.High, false)) + require.False(t, filter.MatchesTemplate("/resolved/normal.yaml", []string{"nginx"}, severity.High, false)) + require.False(t, filter.MatchesTemplate("/resolved/normal.yaml", []string{"apache", "deprecated"}, severity.High, false)) + require.True(t, filter.MatchesTemplate("/resolved/forced.yaml", []string{"deprecated"}, severity.Low, false)) +} + +func TestResolveTargetTemplatePathsRejectsRemoteSelectorsClearly(t *testing.T) { + _, err := resolveTargetTemplatePaths(targetFilterTestCatalog{}, []string{"https://example.com/template.yaml"}, 7, true) + require.ErrorContains(t, err, `jsonl line 7: remote template selector "https://example.com/template.yaml" is not supported`) +} + +func TestResolveTargetTemplatePathsRejectsEscapingLocalSelectors(t *testing.T) { + for _, selector := range []string{"/opt/restricted/rce.yaml", "../../outside-templates/rce.yaml", ".."} { + _, err := resolveTargetTemplatePaths(targetFilterTestCatalog{}, []string{selector}, 4, true) + require.ErrorContains(t, err, "jsonl line 4 template selector") + require.ErrorContains(t, err, "must stay within the templates directory") + } +} + +func TestResolveTargetTemplatePathsAllowsContainedLocalSelectors(t *testing.T) { + paths, err := resolveTargetTemplatePaths(targetFilterTestCatalog{}, []string{"http/cves/2026/", "sub/dir/../ok.yaml"}, 4, true) + require.NoError(t, err) + require.NotEmpty(t, paths) +} + +func TestResolveTargetTemplatePathsAllowsAbsoluteGlobalSelectors(t *testing.T) { + // Global -t/-it selectors are trusted CLI input; absolute paths must still + // resolve when containment is not enforced. + paths, err := resolveTargetTemplatePaths(targetFilterTestCatalog{}, []string{"/opt/custom/global.yaml"}, 0, false) + require.NoError(t, err) + require.NotEmpty(t, paths) +} + +func TestPrepareTargetFiltersRejectsGlobalRemoteURLsWithTemplateOverrides(t *testing.T) { + options := &types.Options{} + options.TemplateURLs = append(options.TemplateURLs, "https://example.com/global.yaml") + runner := &Runner{ + options: options, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + { + Input: "https://example.com", + TargetFilter: &contextargs.TargetFilter{ + HasTemplates: true, + Templates: []string{"local.yaml"}, + }, + }, + }}, + } + + err := runner.prepareTargetFilters(&loader.Config{}) + require.ErrorContains(t, err, "global remote template URLs are not supported together with per-target template overrides") +} + +func TestPrepareTargetFiltersLeavesGlobalBehaviorUntouchedWithoutOverrides(t *testing.T) { + filter := &contextargs.TargetFilter{} + options := &types.Options{} + options.IncludeTemplates = append(options.IncludeTemplates, "https://example.com/include.yaml") + runner := &Runner{ + options: options, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + { + Input: "https://example.com", + TargetFilter: filter, + }, + }}, + } + loaderConfig := &loader.Config{ + Tags: []string{"global"}, + Severities: severity.Severities{severity.High}, + } + + require.NoError(t, runner.prepareTargetFilters(loaderConfig)) + require.Equal(t, []string{"global"}, loaderConfig.Tags) + require.Equal(t, severity.Severities{severity.High}, loaderConfig.Severities) + require.Nil(t, runner.inputProvider.(provider.PerTargetInputProvider).TargetInputs()[0].TargetFilter) +} + +func TestPrepareTargetFiltersRejectsAutomaticScanWithOverrides(t *testing.T) { + runner := &Runner{ + options: &types.Options{AutomaticScan: true}, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + { + Input: "https://example.com", + TargetFilter: &contextargs.TargetFilter{ + HasTags: true, + Tags: []string{"apache"}, + }, + }, + }}, + } + + err := runner.prepareTargetFilters(&loader.Config{}) + require.ErrorContains(t, err, "automatic scan is not supported with per-target JSONL overrides") +} + +func TestPrepareTargetFiltersRejectsWorkflowsWithOverrides(t *testing.T) { + options := &types.Options{} + options.Workflows = append(options.Workflows, "workflow.yaml") + runner := &Runner{ + options: options, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + { + Input: "https://example.com", + TargetFilter: &contextargs.TargetFilter{ + HasSeverities: true, + Severities: severity.Severities{severity.High}, + }, + }, + }}, + } + + err := runner.prepareTargetFilters(&loader.Config{}) + require.ErrorContains(t, err, "workflows are not supported with per-target JSONL overrides") +} + +func TestPrepareTargetFiltersRejectsGlobalMatchersWithOverrides(t *testing.T) { + runner := &Runner{ + options: &types.Options{EnableGlobalMatchersTemplates: true}, + catalog: targetFilterTestCatalog{}, + inputProvider: &targetFilterTestProvider{inputs: []*contextargs.MetaInput{ + { + Input: "https://example.com", + TargetFilter: &contextargs.TargetFilter{ + HasTags: true, + Tags: []string{"apache"}, + }, + }, + }}, + } + + err := runner.prepareTargetFilters(&loader.Config{}) + require.ErrorContains(t, err, "global matchers are not supported with per-target JSONL overrides") +} diff --git a/internal/tests/integration/target_jsonl_test.go b/internal/tests/integration/target_jsonl_test.go new file mode 100644 index 0000000000..7846df5ea3 --- /dev/null +++ b/internal/tests/integration/target_jsonl_test.go @@ -0,0 +1,238 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" + "github.com/projectdiscovery/nuclei/v3/pkg/output" +) + +const ( + targetJSONLAlphaTemplateID = "target-jsonl-alpha" + targetJSONLBetaTemplateID = "target-jsonl-beta" +) + +type targetJSONLRecord struct { + URL string `json:"url"` + Tags []string `json:"tags,omitempty"` + ExcludeTags []string `json:"exclude-tags,omitempty"` + Templates *[]string `json:"templates,omitempty"` +} + +func TestTargetJSONLTemplateFilters(t *testing.T) { + templatePaths := writeTargetJSONLTemplates(t) + + scenarios := []struct { + name string + records func(firstURL, secondURL string) []targetJSONLRecord + expected func(firstURL, secondURL string) map[string]struct{} + }{ + { + name: "clustered templates", + records: func(firstURL, secondURL string) []targetJSONLRecord { + return []targetJSONLRecord{ + {URL: firstURL, Tags: []string{targetJSONLAlphaTemplateID, targetJSONLBetaTemplateID}}, + {URL: secondURL, Tags: []string{targetJSONLAlphaTemplateID, targetJSONLBetaTemplateID}}, + } + }, + expected: func(firstURL, secondURL string) map[string]struct{} { + return expectedTargetJSONLPairs( + targetJSONLAlphaTemplateID, firstURL, + targetJSONLBetaTemplateID, firstURL, + targetJSONLAlphaTemplateID, secondURL, + targetJSONLBetaTemplateID, secondURL, + ) + }, + }, + { + name: "target exclusions", + records: func(firstURL, secondURL string) []targetJSONLRecord { + return []targetJSONLRecord{ + { + URL: firstURL, + Tags: []string{targetJSONLAlphaTemplateID, targetJSONLBetaTemplateID}, + ExcludeTags: []string{targetJSONLBetaTemplateID}, + }, + { + URL: secondURL, + Tags: []string{targetJSONLAlphaTemplateID, targetJSONLBetaTemplateID}, + ExcludeTags: []string{targetJSONLAlphaTemplateID}, + }, + } + }, + expected: func(firstURL, secondURL string) map[string]struct{} { + return expectedTargetJSONLPairs( + targetJSONLAlphaTemplateID, firstURL, + targetJSONLBetaTemplateID, secondURL, + ) + }, + }, + { + name: "per-target templates", + records: func(firstURL, secondURL string) []targetJSONLRecord { + firstTemplates := []string{templatePaths[0]} + secondTemplates := []string{templatePaths[1]} + return []targetJSONLRecord{ + {URL: firstURL, Templates: &firstTemplates}, + {URL: secondURL, Templates: &secondTemplates}, + } + }, + expected: func(firstURL, secondURL string) map[string]struct{} { + return expectedTargetJSONLPairs( + targetJSONLAlphaTemplateID, firstURL, + targetJSONLBetaTemplateID, secondURL, + ) + }, + }, + } + + for _, strategy := range []string{"template-spray", "host-spray"} { + strategy := strategy + t.Run(strategy, func(t *testing.T) { + for _, scenario := range scenarios { + scenario := scenario + t.Run(scenario.name, func(t *testing.T) { + var requestCount atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) + _, _ = fmt.Fprint(w, "target-jsonl-integration") + })) + t.Cleanup(server.Close) + + firstURL := server.URL + "/first" + secondURL := server.URL + "/second" + inputPath := writeTargetJSONLInput(t, scenario.records(firstURL, secondURL)) + + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-list", inputPath, + "-input-mode", "jsonl", + "-templates", templatePaths[0], + "-templates", templatePaths[1], + "-jsonl", + "-scan-strategy", strategy, + ) + if err != nil { + t.Fatalf("target JSONL scan failed: %v", err) + } + + assertTargetJSONLResults(t, results, scenario.expected(firstURL, secondURL)) + if got := requestCount.Load(); got != 2 { + t.Fatalf("unexpected physical request count: got %d, want 2 (one clustered request per target)", got) + } + }) + } + }) + } +} + +func writeTargetJSONLTemplates(t *testing.T) [2]string { + t.Helper() + + templateDir := t.TempDir() + paths := [2]string{ + filepath.Join(templateDir, targetJSONLAlphaTemplateID+".yaml"), + filepath.Join(templateDir, targetJSONLBetaTemplateID+".yaml"), + } + for index, templateID := range []string{targetJSONLAlphaTemplateID, targetJSONLBetaTemplateID} { + writeTargetJSONLTemplate(t, paths[index], templateID) + } + return paths +} + +func writeTargetJSONLTemplate(t *testing.T, path, templateID string) { + t.Helper() + + template := fmt.Sprintf(`id: %s + +info: + name: Target JSONL integration %s + author: pdteam + severity: info + tags: %s + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "target-jsonl-integration" +`, templateID, strings.TrimPrefix(templateID, "target-jsonl-"), templateID) + if err := os.WriteFile(path, []byte(template), 0600); err != nil { + t.Fatalf("failed to write integration template %s: %v", templateID, err) + } +} + +func writeTargetJSONLInput(t *testing.T, records []targetJSONLRecord) string { + t.Helper() + + var data strings.Builder + encoder := json.NewEncoder(&data) + for _, record := range records { + if err := encoder.Encode(record); err != nil { + t.Fatalf("failed to encode target JSONL record: %v", err) + } + } + + inputPath := filepath.Join(t.TempDir(), "targets.jsonl") + if err := os.WriteFile(inputPath, []byte(data.String()), 0600); err != nil { + t.Fatalf("failed to write target JSONL input: %v", err) + } + return inputPath +} + +func expectedTargetJSONLPairs(values ...string) map[string]struct{} { + pairs := make(map[string]struct{}, len(values)/2) + for index := 0; index < len(values); index += 2 { + pairs[targetJSONLPair(values[index], values[index+1])] = struct{}{} + } + return pairs +} + +func targetJSONLPair(templateID, targetURL string) string { + return templateID + "|" + targetURL +} + +func assertTargetJSONLResults(t *testing.T, results []string, expected map[string]struct{}) { + t.Helper() + + actual := make(map[string]struct{}, len(results)) + for _, result := range results { + var event output.ResultEvent + if err := json.Unmarshal([]byte(result), &event); err != nil { + t.Fatalf("failed to decode JSONL result %q: %v", result, err) + } + pair := targetJSONLPair(event.TemplateID, event.URL) + if _, duplicate := actual[pair]; duplicate { + t.Fatalf("duplicate result for %s", pair) + } + actual[pair] = struct{}{} + } + + if len(actual) != len(expected) { + t.Fatalf("unexpected result count: got %d, want %d\nresults: %v", len(actual), len(expected), actual) + } + for pair := range expected { + if _, ok := actual[pair]; !ok { + t.Errorf("missing result for %s", pair) + } + } + for pair := range actual { + if _, ok := expected[pair]; !ok { + t.Errorf("unexpected result for %s", pair) + } + } +} diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go index 393aabb8d0..68105d14fc 100644 --- a/pkg/core/execute_options.go +++ b/pkg/core/execute_options.go @@ -81,7 +81,12 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ // Filter Self Contained templates since they are not bound to target for _, v := range finalTemplates { if v.SelfContained { - selfContained = append(selfContained, v) + if templateMatchesAnyTarget(v, target) { + selfContained = append(selfContained, v) + e.adjustSelfContainedProgress(v, target, true) + } else { + e.adjustSelfContainedProgress(v, target, false) + } } else { filtered = append(filtered, v) } @@ -104,6 +109,31 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ return results } +func (e *Engine) adjustSelfContainedProgress(template *templates.Template, target provider.InputProvider, selected bool) { + if target.InputType() != provider.TargetInputProvider || e.executerOpts.Progress == nil || template.TotalRequests == 0 { + return + } + skippedExecutions := target.Count() + if selected && skippedExecutions > 0 { + skippedExecutions-- + } + if skippedExecutions > 0 { + e.executerOpts.Progress.AddToTotal(-int64(template.TotalRequests) * skippedExecutions) + } +} + +func templateMatchesAnyTarget(template *templates.Template, target provider.InputProvider) bool { + if target.InputType() != provider.TargetInputProvider { + return true + } + matched := false + target.Iterate(func(input *contextargs.MetaInput) bool { + matched = templateMatchesTarget(template, input) + return !matched + }) + return matched +} + // executeTemplateSpray executes scan using template spray strategy where targets are iterated over each template func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool { results := &atomic.Bool{} diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 8b2c47559a..ffa03b7690 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -125,6 +125,12 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ default: } + if !templateMatchesTarget(template, scannedValue) { + e.noteFilteredTemplate(template) + index++ + return true + } + // Best effort to track the host progression // skips indexes lower than the minimum in-flight at interruption time var skip bool @@ -199,6 +205,11 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t default: } + if !templateMatchesTarget(tpl, target) { + e.noteFilteredTemplate(tpl) + continue + } + // Check whether the target has already been marked as permanently // unresponsive by HostErrorsCache before spawning another goroutine. if e.executerOpts.HostErrorsCache != nil && @@ -245,6 +256,10 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t // executeTemplateOnInput performs template execution for a single input and returns match status and error func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates.Template, value *contextargs.MetaInput) (bool, error) { + if !templateMatchesTarget(template, value) { + return false, nil + } + ctxArgs := contextargs.New(ctx) ctxArgs.MetaInput = value scanCtx := scan.NewScanContext(ctx, ctxArgs) @@ -266,3 +281,28 @@ func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates return template.Executer.Execute(scanCtx) } } + +type targetFilterMatcher interface { + MatchesTargetFilter(filter *contextargs.TargetFilter) bool +} + +func templateMatchesTarget(template *templates.Template, input *contextargs.MetaInput) bool { + if input == nil || input.TargetFilter == nil { + return true + } + if matcher, ok := template.Executer.(targetFilterMatcher); ok { + return matcher.MatchesTargetFilter(input.TargetFilter) + } + return input.TargetFilter.MatchesTemplate( + template.Path, + template.Info.Tags.ToSlice(), + template.Info.SeverityHolder.Severity, + template.Type() == types.WorkflowProtocol, + ) +} + +func (e *Engine) noteFilteredTemplate(template *templates.Template) { + if e.executerOpts.Progress != nil && template.TotalRequests > 0 { + e.executerOpts.Progress.AddToTotal(-int64(template.TotalRequests)) + } +} diff --git a/pkg/core/executors_test.go b/pkg/core/executors_test.go index 394b2e6d97..fb4bb613f7 100644 --- a/pkg/core/executors_test.go +++ b/pkg/core/executors_test.go @@ -8,6 +8,9 @@ import ( "time" inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" @@ -15,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/templates" tmpltypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/stretchr/testify/require" ) // fakeExecuter is a simple stub for protocols.Executer used to test executeTemplateOnInput @@ -34,7 +38,12 @@ func (f *fakeExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.Resu // newTestEngine creates a minimal Engine for tests func newTestEngine() *Engine { - return New(&types.Options{}) + return New(&types.Options{ + BulkSize: 1, + TemplateThreads: 1, + HeadlessBulkSize: 1, + HeadlessTemplateThreads: 1, + }) } func Test_executeTemplateOnInput_CallbackPath(t *testing.T) { @@ -96,7 +105,8 @@ func Test_executeTemplateOnInput_CallbackErrorPropagates(t *testing.T) { } type fakeTargetProvider struct { - values []*contextargs.MetaInput + values []*contextargs.MetaInput + inputType string } func (f *fakeTargetProvider) Count() int64 { return int64(len(f.values)) } @@ -112,8 +122,13 @@ func (f *fakeTargetProvider) SetWithProbe(string, string, inputtypes.InputLivene return nil } func (f *fakeTargetProvider) SetWithExclusions(string, string) error { return nil } -func (f *fakeTargetProvider) InputType() string { return "test" } -func (f *fakeTargetProvider) Close() {} +func (f *fakeTargetProvider) InputType() string { + if f.inputType != "" { + return f.inputType + } + return "test" +} +func (f *fakeTargetProvider) Close() {} type slowExecuter struct{} @@ -146,3 +161,140 @@ func Test_executeTemplateWithTargets_RespectsCancellation(t *testing.T) { var matched atomic.Bool e.executeTemplateWithTargets(ctx, tpl, targets, &matched) } + +type countingExecuter struct { + executions atomic.Int32 +} + +func (c *countingExecuter) Compile() error { return nil } +func (c *countingExecuter) Requests() int { return 1 } +func (c *countingExecuter) Execute(*scan.ScanContext) (bool, error) { + c.executions.Add(1) + return false, nil +} +func (c *countingExecuter) ExecuteWithResults(*scan.ScanContext) ([]*output.ResultEvent, error) { + c.executions.Add(1) + return nil, nil +} + +type progressSpy struct { + totalDelta atomic.Int64 +} + +func (*progressSpy) Stop() {} +func (*progressSpy) Init(int64, int, int64) {} +func (p *progressSpy) AddToTotal(delta int64) { p.totalDelta.Add(delta) } +func (*progressSpy) IncrementRequests() {} +func (*progressSpy) SetRequests(uint64) {} +func (*progressSpy) IncrementMatched() {} +func (*progressSpy) IncrementErrorsBy(int64) {} +func (*progressSpy) IncrementFailedRequestsBy(int64) {} + +func TestExecuteTemplateWithTargetsAppliesPerTargetFilter(t *testing.T) { + e := newTestEngine() + progress := &progressSpy{} + e.SetExecuterOptions(&protocols.ExecutorOptions{ + Logger: e.Logger, + ResumeCfg: types.NewResumeCfg(), + ProtocolType: tmpltypes.HTTPProtocol, + Progress: progress, + }) + + executer := &countingExecuter{} + template := filteredTestTemplate("apache.yaml", "apache", severity.High, executer) + allowed := preparedMetaInput("https://allowed.example", []string{"apache"}) + denied := preparedMetaInput("https://denied.example", []string{"nginx"}) + targets := &fakeTargetProvider{values: []*contextargs.MetaInput{denied, allowed}} + + var matched atomic.Bool + e.executeTemplateWithTargets(context.Background(), template, targets, &matched) + + require.EqualValues(t, 1, executer.executions.Load()) + require.EqualValues(t, -1, progress.totalDelta.Load()) +} + +func TestExecuteTemplatesOnTargetAppliesPerTargetFilter(t *testing.T) { + e := newTestEngine() + progress := &progressSpy{} + e.SetExecuterOptions(&protocols.ExecutorOptions{ + Logger: e.Logger, + ResumeCfg: types.NewResumeCfg(), + ProtocolType: tmpltypes.HTTPProtocol, + Progress: progress, + }) + + allowedExecuter := &countingExecuter{} + deniedExecuter := &countingExecuter{} + templatesList := []*templates.Template{ + filteredTestTemplate("apache.yaml", "apache", severity.High, allowedExecuter), + filteredTestTemplate("nginx.yaml", "nginx", severity.High, deniedExecuter), + } + target := preparedMetaInput("https://example.com", []string{"apache"}) + + var matched atomic.Bool + e.executeTemplatesOnTarget(context.Background(), templatesList, target, &matched) + + require.EqualValues(t, 1, allowedExecuter.executions.Load()) + require.Zero(t, deniedExecuter.executions.Load()) + require.EqualValues(t, -1, progress.totalDelta.Load()) +} + +func TestTemplateMatchesAnyTargetForSelfContainedTemplate(t *testing.T) { + executer := &countingExecuter{} + template := filteredTestTemplate("apache.yaml", "apache", severity.High, executer) + template.SelfContained = true + + deniedOnly := &fakeTargetProvider{ + inputType: "TargetInputProvider", + values: []*contextargs.MetaInput{preparedMetaInput("https://denied.example", []string{"nginx"})}, + } + require.False(t, templateMatchesAnyTarget(template, deniedOnly)) + + withAllowed := &fakeTargetProvider{ + inputType: "TargetInputProvider", + values: []*contextargs.MetaInput{ + preparedMetaInput("https://denied.example", []string{"nginx"}), + preparedMetaInput("https://allowed.example", []string{"apache"}), + }, + } + require.True(t, templateMatchesAnyTarget(template, withAllowed)) +} + +func TestAdjustSelfContainedProgressForTargetFilters(t *testing.T) { + e := newTestEngine() + progress := &progressSpy{} + e.SetExecuterOptions(&protocols.ExecutorOptions{Progress: progress}) + template := filteredTestTemplate("self-contained.yaml", "code", severity.Info, &countingExecuter{}) + targets := &fakeTargetProvider{ + inputType: "TargetInputProvider", + values: []*contextargs.MetaInput{ + preparedMetaInput("https://one.example", []string{"code"}), + preparedMetaInput("https://two.example", []string{"code"}), + }, + } + + e.adjustSelfContainedProgress(template, targets, true) + require.EqualValues(t, -1, progress.totalDelta.Load()) + + progress.totalDelta.Store(0) + e.adjustSelfContainedProgress(template, targets, false) + require.EqualValues(t, -2, progress.totalDelta.Load()) +} + +func filteredTestTemplate(path, tag string, templateSeverity severity.Severity, executer protocols.Executer) *templates.Template { + return &templates.Template{ + Path: path, + Info: model.Info{ + Tags: stringslice.New([]string{tag}), + SeverityHolder: severity.Holder{Severity: templateSeverity}, + }, + Executer: executer, + TotalRequests: 1, + } +} + +func preparedMetaInput(input string, tags []string) *contextargs.MetaInput { + filter := &contextargs.TargetFilter{} + filter.Prepare(tags, nil, nil, nil, nil, false) + return &contextargs.MetaInput{Input: input, TargetFilter: filter} +} diff --git a/pkg/input/README.md b/pkg/input/README.md index b5936c5803..d8f2e1867d 100644 --- a/pkg/input/README.md +++ b/pkg/input/README.md @@ -27,3 +27,36 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) This function returns a InputProvider based by appropriately selecting input provider based on the input format (i.e. either list or http) and returns the provider that can handle that input format. +## Per-target JSONL filters + +JSONL input can attach template-selection overrides to individual URL targets: + +```jsonl +{"url":"https://target-a.example","tags":["apache","shiro"],"severity":["critical","high"]} +{"url":"https://target-b.example","exclude-tags":["tomcat"],"templates":["http/cves/2026/"]} +{"url":"https://target-c.example"} +``` + +Run the scan with: + +```console +nuclei -l targets.jsonl -input-mode jsonl +``` + +The optional `tags`, `exclude-tags`, `severity`, and `templates` fields mirror +their global CLI counterparts. An omitted field inherits the global option; +an explicitly empty array clears that option for the target. Global +`-include-templates` selections remain forced includes, and `-exclude-hosts` +is applied before target execution. Two global exclusions always stay in +effect: built-in ignore-file tags are not re-enabled by a target-level +`exclude-tags`, and `-exclude-severity` cannot be bypassed by a target-level +`severity`, even when those target fields are explicitly empty. + +Per-target `templates` overrides currently support local selectors only and +cannot be combined with global remote templates. Each selector must stay within +the templates directory: absolute paths and `../` parent-directory traversal are +rejected, so a targets file cannot point the loader at arbitrary files on disk. +Use the global `-t` flag for templates outside the templates tree. Any per-target override is +incompatible with automatic scan, workflows, and global matchers. +JSONL files in the existing Proxify request/response format remain supported, +but target and Proxify records cannot be mixed in one file. diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index 9de4d0d013..b5dc9882a7 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/retryablehttp-go" fileutil "github.com/projectdiscovery/utils/file" "gopkg.in/yaml.v3" @@ -15,6 +16,10 @@ import ( // ParseReqRespCallback is a callback function for discovered raw requests type ParseReqRespCallback func(rr *types.RequestResponse) bool +// ParseMetaInputCallback is a callback for formats that can provide complete +// MetaInput values rather than only raw HTTP request/response pairs. +type ParseMetaInputCallback func(input *contextargs.MetaInput) bool + // InputFormatOptions contains options for the input // this can be variables that can be passed or // overrides or some other options @@ -48,6 +53,13 @@ type Format interface { SetOptions(options InputFormatOptions) } +// MetaInputFormat is implemented by input formats that can attach metadata to +// individual targets. Existing request/response formats continue to implement +// Format only. +type MetaInputFormat interface { + ParseMeta(input io.Reader, resultsCb ParseMetaInputCallback, filePath string) error +} + // SpecDownloader is an interface for downloading API specifications from URLs type SpecDownloader interface { // Download downloads the spec from the given URL and saves it to tmpDir diff --git a/pkg/input/formats/json/json.go b/pkg/input/formats/json/json.go index fdf05aa152..a1057e8b7a 100644 --- a/pkg/input/formats/json/json.go +++ b/pkg/input/formats/json/json.go @@ -1,12 +1,19 @@ package json import ( + "bytes" + stdjson "encoding/json" + "fmt" "io" + "sort" + "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" ) @@ -22,6 +29,7 @@ func New() *JSONFormat { } var _ formats.Format = &JSONFormat{} +var _ formats.MetaInputFormat = &JSONFormat{} // proxifyRequest is a request for proxify type proxifyRequest struct { @@ -34,6 +42,14 @@ type proxifyRequest struct { } `json:"request"` } +type targetRequest struct { + URL string `json:"url"` + Tags *[]string `json:"tags"` + ExcludeTags *[]string `json:"exclude-tags"` + Severity *[]string `json:"severity"` + Templates *[]string `json:"templates"` +} + // Name returns the name of the format func (j *JSONFormat) Name() string { return "jsonl" @@ -69,3 +85,206 @@ func (j *JSONFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallba } return nil } + +// ParseMeta parses either legacy Proxify JSONL records or per-target JSONL +// records. Mixing both record kinds in the same file is rejected because they +// require different execution modes. +func (j *JSONFormat) ParseMeta(input io.Reader, resultsCb formats.ParseMetaInputCallback, _ string) error { + tracker := &lineTrackingReader{reader: input} + decoder := stdjson.NewDecoder(tracker) + recordKind := "" + + for { + var raw stdjson.RawMessage + if err := decoder.Decode(&raw); err != nil { + if err == io.EOF { + break + } + lineNumber := tracker.lineAt(decoder.InputOffset()) + if syntaxErr, ok := err.(*stdjson.SyntaxError); ok { + lineNumber = tracker.lineAt(syntaxErr.Offset - 1) + } + return fmt.Errorf("jsonl line %d: invalid JSON: %w", lineNumber, err) + } + lineNumber := tracker.lineAt(decoder.InputOffset() - int64(len(raw))) + + var fields map[string]stdjson.RawMessage + if err := stdjson.Unmarshal(raw, &fields); err != nil { + return fmt.Errorf("jsonl line %d: each record must be a JSON object: %w", lineNumber, err) + } + + kind := recordType(fields) + if kind == "proxify" { + for _, name := range []string{"tags", "exclude-tags", "severity", "templates"} { + if _, ok := fields[name]; ok { + return fmt.Errorf("jsonl line %d: Proxify request records cannot include target filter field %q", lineNumber, name) + } + } + } + if recordKind == "" { + recordKind = kind + } else if recordKind != kind { + return fmt.Errorf("jsonl line %d: cannot mix %s and %s records in one input file", lineNumber, recordKind, kind) + } + + var metaInput *contextargs.MetaInput + var err error + switch kind { + case "proxify": + metaInput, err = parseProxifyMetaInput(raw) + default: + metaInput, err = parseTargetMetaInput(raw, fields, lineNumber) + } + if err != nil { + return err + } + if metaInput != nil && !resultsCb(metaInput) { + return nil + } + } + return nil +} + +type lineTrackingReader struct { + reader io.Reader + offset int64 + newlines []int64 +} + +func (r *lineTrackingReader) Read(data []byte) (int, error) { + read, err := r.reader.Read(data) + for index, value := range data[:read] { + if value == '\n' { + r.newlines = append(r.newlines, r.offset+int64(index)) + } + } + r.offset += int64(read) + return read, err +} + +func (r *lineTrackingReader) lineAt(offset int64) int { + if offset < 0 { + offset = 0 + } + return sort.Search(len(r.newlines), func(index int) bool { + return r.newlines[index] >= offset + }) + 1 +} + +func recordType(fields map[string]stdjson.RawMessage) string { + rawRequest, ok := fields["request"] + if !ok { + return "target" + } + var requestFields map[string]stdjson.RawMessage + if err := stdjson.Unmarshal(rawRequest, &requestFields); err != nil { + return "target" + } + for _, name := range []string{"header", "body", "raw", "endpoint"} { + if _, ok := requestFields[name]; ok { + return "proxify" + } + } + return "target" +} + +func parseProxifyMetaInput(data []byte) (*contextargs.MetaInput, error) { + var request proxifyRequest + if err := stdjson.Unmarshal(data, &request); err != nil { + return nil, fmt.Errorf("could not decode Proxify JSONL record: %w", err) + } + if request.URL == "" && request.Request.Endpoint != "" { + request.URL = request.Request.Endpoint + } + rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL) + if err != nil { + gologger.Warning().Msgf("jsonl: Could not parse raw request %s: %s\n", request.URL, err) + return nil, nil + } + metaInput := contextargs.NewMetaInput() + metaInput.ReqResp = rawRequest + metaInput.Input = rawRequest.URL.String() + return metaInput, nil +} + +func parseTargetMetaInput(data []byte, fields map[string]stdjson.RawMessage, lineNumber int) (*contextargs.MetaInput, error) { + for name, raw := range fields { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil, fmt.Errorf("jsonl line %d: field %q must not be null", lineNumber, name) + } + } + + decoder := stdjson.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var target targetRequest + if err := decoder.Decode(&target); err != nil { + return nil, fmt.Errorf("jsonl line %d: invalid target record: %w", lineNumber, err) + } + if strings.TrimSpace(target.URL) == "" { + return nil, fmt.Errorf("jsonl line %d: field %q is required and must not be empty", lineNumber, "url") + } + + filter := &contextargs.TargetFilter{SourceLine: lineNumber} + var err error + if target.Tags != nil { + filter.HasTags = true + filter.Tags, err = normalizeValues(*target.Tags, true, "tags", lineNumber) + if err != nil { + return nil, err + } + } + if target.ExcludeTags != nil { + filter.HasExcludeTags = true + filter.ExcludeTags, err = normalizeValues(*target.ExcludeTags, true, "exclude-tags", lineNumber) + if err != nil { + return nil, err + } + } + if target.Severity != nil { + filter.HasSeverities = true + normalized, normalizeErr := normalizeValues(*target.Severity, true, "severity", lineNumber) + if normalizeErr != nil { + return nil, normalizeErr + } + for _, value := range normalized { + encoded, _ := stdjson.Marshal(value) + var holder severity.Holder + if err := holder.UnmarshalJSON(encoded); err != nil { + return nil, fmt.Errorf("jsonl line %d: invalid severity %q: %w", lineNumber, value, err) + } + filter.Severities = append(filter.Severities, holder.Severity) + } + } + if target.Templates != nil { + filter.HasTemplates = true + filter.Templates, err = normalizeValues(*target.Templates, false, "templates", lineNumber) + if err != nil { + return nil, err + } + } + + metaInput := contextargs.NewMetaInput() + metaInput.Input = strings.TrimSpace(target.URL) + metaInput.TargetFilter = filter + return metaInput, nil +} + +func normalizeValues(values []string, lowercase bool, field string, lineNumber int) ([]string, error) { + normalized := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for index, value := range values { + value = strings.TrimSpace(value) + if value == "" { + return nil, fmt.Errorf("jsonl line %d: field %q contains an empty value at index %d", lineNumber, field, index) + } + if lowercase { + value = strings.ToLower(value) + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + normalized = append(normalized, value) + } + return normalized, nil +} diff --git a/pkg/input/formats/json/json_test.go b/pkg/input/formats/json/json_test.go index 830385e213..7347a7b7b8 100644 --- a/pkg/input/formats/json/json_test.go +++ b/pkg/input/formats/json/json_test.go @@ -2,9 +2,11 @@ package json import ( "os" + "strings" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/stretchr/testify/require" ) @@ -62,3 +64,145 @@ func TestJSONFormatterParse(t *testing.T) { } require.ElementsMatch(t, urls, expectedURLs) } + +func TestJSONFormatterParseTargetRecords(t *testing.T) { + const input = `{"url":"https://one.example","tags":[" Apache ","apache","Shiro"],"exclude-tags":[],"severity":["HIGH"],"templates":[" cves/2026/ "]} +{"url":"https://two.example"} +` + + var inputs []*contextargs.MetaInput + err := New().ParseMeta(strings.NewReader(input), func(input *contextargs.MetaInput) bool { + inputs = append(inputs, input) + return true + }, "") + require.NoError(t, err) + require.Len(t, inputs, 2) + + first := inputs[0] + require.Equal(t, "https://one.example", first.Input) + require.True(t, first.TargetFilter.HasTags) + require.True(t, first.TargetFilter.HasExcludeTags) + require.True(t, first.TargetFilter.HasSeverities) + require.True(t, first.TargetFilter.HasTemplates) + require.Equal(t, []string{"apache", "shiro"}, first.TargetFilter.Tags) + require.Empty(t, first.TargetFilter.ExcludeTags) + require.Equal(t, "high", first.TargetFilter.Severities.String()) + require.Equal(t, []string{"cves/2026/"}, first.TargetFilter.Templates) + + second := inputs[1] + require.Equal(t, "https://two.example", second.Input) + require.False(t, second.TargetFilter.HasTags) + require.False(t, second.TargetFilter.HasExcludeTags) + require.False(t, second.TargetFilter.HasSeverities) + require.False(t, second.TargetFilter.HasTemplates) +} + +func TestJSONFormatterTreatsSeverityAsLiteralValue(t *testing.T) { + previousWorkingDirectory, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(t.TempDir())) + t.Cleanup(func() { + require.NoError(t, os.Chdir(previousWorkingDirectory)) + }) + require.NoError(t, os.WriteFile("high", []byte("critical\n"), 0o600)) + + const input = `{"url":"https://example.com","severity":["high"]}` + var parsed *contextargs.MetaInput + err = New().ParseMeta(strings.NewReader(input), func(input *contextargs.MetaInput) bool { + parsed = input + return true + }, "") + + require.NoError(t, err) + require.NotNil(t, parsed) + require.Equal(t, "high", parsed.TargetFilter.Severities.String()) +} + +func TestJSONFormatterParseTargetValidation(t *testing.T) { + tests := []struct { + name string + input string + message string + }{ + {name: "missing URL", input: `{"tags":["apache"]}`, message: `field "url" is required`}, + {name: "empty URL", input: `{"url":" "}`, message: `field "url" is required`}, + {name: "null field", input: `{"url":"https://example.com","tags":null}`, message: `field "tags" must not be null`}, + {name: "unknown field", input: `{"url":"https://example.com","unknown":true}`, message: `unknown field "unknown"`}, + {name: "request is not a target field", input: `{"url":"https://example.com","request":{}}`, message: `unknown field "request"`}, + {name: "wrong tags type", input: `{"url":"https://example.com","tags":"apache"}`, message: `cannot unmarshal string`}, + {name: "empty list value", input: `{"url":"https://example.com","tags":[""]}`, message: `contains an empty value`}, + {name: "invalid severity", input: `{"url":"https://example.com","severity":["urgent"]}`, message: `invalid severity "urgent"`}, + {name: "not an object", input: `["https://example.com"]`, message: `each record must be a JSON object`}, + {name: "malformed JSON", input: "{\n", message: `jsonl line 1: invalid JSON`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := New().ParseMeta(strings.NewReader(test.input), func(*contextargs.MetaInput) bool { + return true + }, "") + require.ErrorContains(t, err, test.message) + }) + } +} + +func TestJSONFormatterParseMetaStopsOnCallback(t *testing.T) { + const input = `{"url":"https://one.example"} +{"url":"https://two.example"} +` + count := 0 + err := New().ParseMeta(strings.NewReader(input), func(*contextargs.MetaInput) bool { + count++ + return false + }, "") + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestJSONFormatterParseMetaRejectsMixedRecords(t *testing.T) { + const input = `{"url":"https://one.example"} +{"url":"https://two.example","request":{"raw":"GET / HTTP/1.1\r\nHost: two.example\r\n\r\n"}} +` + err := New().ParseMeta(strings.NewReader(input), func(*contextargs.MetaInput) bool { + return true + }, "") + require.ErrorContains(t, err, "cannot mix target and proxify records") +} + +func TestJSONFormatterParseMetaRejectsHybridProxifyRecord(t *testing.T) { + const input = `{"url":"https://example.com","tags":["apache"],"request":{"raw":"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"}}` + err := New().ParseMeta(strings.NewReader(input), func(*contextargs.MetaInput) bool { + return true + }, "") + require.ErrorContains(t, err, `jsonl line 1: Proxify request records cannot include target filter field "tags"`) +} + +func TestJSONFormatterReportsRecordLineNumbers(t *testing.T) { + const input = `{"url":"https://one.example"} + +{"url":"https://two.example","unknown":true} +` + err := New().ParseMeta(strings.NewReader(input), func(*contextargs.MetaInput) bool { + return true + }, "") + require.ErrorContains(t, err, `jsonl line 3: invalid target record`) +} + +func TestJSONFormatterParseMetaPreservesMultilineProxifyInput(t *testing.T) { + const input = `{ + "url": "https://example.com/", + "request": { + "raw": "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n" + } +}` + var inputs []*contextargs.MetaInput + err := New().ParseMeta(strings.NewReader(input), func(input *contextargs.MetaInput) bool { + inputs = append(inputs, input) + return true + }, "") + require.NoError(t, err) + require.Len(t, inputs, 1) + require.Equal(t, "https://example.com/", inputs[0].Input) + require.NotNil(t, inputs[0].ReqResp) + require.Nil(t, inputs[0].TargetFilter) +} diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index ee8cb68093..92fe9115e4 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -4,10 +4,13 @@ import ( "bytes" "io" "os" + "regexp" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/mapcidr/asn" + "github.com/projectdiscovery/networkpolicy" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json" @@ -16,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/yaml" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + urlutil "github.com/projectdiscovery/utils/url" ) // HttpMultiFormatOptions contains options for the http input provider @@ -29,15 +33,24 @@ type HttpMultiFormatOptions struct { // optional input reader InputContents string + + // ExcludeTargets contains URL, host, IP, CIDR, or ASN exclusions to + // apply to URL target records. Request/response input formats retain their + // existing protocol-level exclusion behavior. + ExcludeTargets []string } +const targetInputProviderType = "TargetInputProvider" + // HttpInputProvider implements an input provider for nuclei that loads // inputs from multiple formats like burp, openapi, postman,proxify, etc. type HttpInputProvider struct { - format formats.Format - inputData []byte - inputFile string - count int64 + format formats.Format + inputData []byte + inputFile string + count int64 + inputType string + targetInputs []*contextargs.MetaInput } // NewHttpInputProvider creates a new input provider for nuclei from a file @@ -85,14 +98,57 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err return nil, errors.New("input file is empty") } - parseErr := format.Parse(bytes.NewReader(data), func(request *types.RequestResponse) bool { + var exclusionPolicy *networkpolicy.NetworkPolicy + var exclusionPolicyErr error + exclusionPolicyPrepared := false + targetInputs := make([]*contextargs.MetaInput, 0) + hasTargetRecords := false + excludedCount := int64(0) + parseErr := parseFormat(format, bytes.NewReader(data), opts.InputFile, func(metaInput *contextargs.MetaInput) bool { + if metaInput == nil { + return true + } + if metaInput.TargetFilter != nil { + hasTargetRecords = true + if !exclusionPolicyPrepared { + exclusionPolicy, exclusionPolicyErr = newTargetExclusionPolicy(opts.ExcludeTargets) + exclusionPolicyPrepared = true + if exclusionPolicyErr != nil { + return false + } + } + if targetIsExcluded(metaInput.Input, exclusionPolicy) { + excludedCount++ + return true + } + targetInputs = append(targetInputs, metaInput) + } count++ - return false - }, opts.InputFile) + return true + }) + if exclusionPolicyErr != nil { + return nil, errors.Wrap(exclusionPolicyErr, "could not prepare target exclusions") + } if parseErr != nil { return nil, errors.Wrap(parseErr, "could not parse input file") } - return &HttpInputProvider{format: format, inputData: data, inputFile: opts.InputFile, count: count}, nil + if excludedCount > 0 { + gologger.Info().Msgf("Number of JSONL targets excluded from input: %d", excludedCount) + } + inputType := providerInputType(format, hasTargetRecords) + if inputType == targetInputProviderType { + // Target records are cached as parsed MetaInputs, so retaining the raw + // file would duplicate memory for large JSONL target sets. + data = nil + } + return &HttpInputProvider{ + format: format, + inputData: data, + inputFile: opts.InputFile, + count: count, + inputType: inputType, + targetInputs: targetInputs, + }, nil } // Count returns the number of items for input provider @@ -102,12 +158,16 @@ func (i *HttpInputProvider) Count() int64 { // Iterate over all inputs in order func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { - err := i.format.Parse(bytes.NewReader(i.inputData), func(request *types.RequestResponse) bool { - metaInput := contextargs.NewMetaInput() - metaInput.ReqResp = request - metaInput.Input = request.URL.String() - return callback(metaInput) - }, i.inputFile) + if i.inputType == targetInputProviderType { + for _, input := range i.targetInputs { + if !callback(input.Clone()) { + return + } + } + return + } + + err := parseFormat(i.format, bytes.NewReader(i.inputData), i.inputFile, callback) if err != nil { gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err) } @@ -131,7 +191,14 @@ func (i *HttpInputProvider) SetWithExclusions(_ string, value string) error { // InputType returns the type of input provider func (i *HttpInputProvider) InputType() string { - return "MultiFormatInputProvider" + return i.inputType +} + +// TargetInputs returns the cached target records for inheritance preparation. +// The returned inputs are owned by the provider and must only be mutated before +// scan execution starts. +func (i *HttpInputProvider) TargetInputs() []*contextargs.MetaInput { + return i.targetInputs } // Close closes the input provider and cleans up any resources @@ -156,3 +223,71 @@ func SupportedFormats() string { } return strings.Join(formats, ", ") } + +func parseFormat(format formats.Format, input io.Reader, filePath string, callback formats.ParseMetaInputCallback) error { + if metaFormat, ok := format.(formats.MetaInputFormat); ok { + return metaFormat.ParseMeta(input, callback, filePath) + } + return format.Parse(input, func(request *types.RequestResponse) bool { + metaInput := contextargs.NewMetaInput() + metaInput.ReqResp = request + metaInput.Input = request.URL.String() + return callback(metaInput) + }, filePath) +} + +func providerInputType(format formats.Format, hasTargetRecords bool) string { + if _, ok := format.(formats.MetaInputFormat); ok && hasTargetRecords { + return targetInputProviderType + } + return "MultiFormatInputProvider" +} + +func newTargetExclusionPolicy(excludeTargets []string) (*networkpolicy.NetworkPolicy, error) { + if len(excludeTargets) == 0 { + return nil, nil + } + + denyList := make([]string, 0, len(excludeTargets)) + for _, target := range excludeTargets { + target = strings.TrimSpace(target) + if target == "" { + continue + } + if asn.IsASN(target) { + cidrs, err := asn.GetCIDRsForASNNum(target) + if err != nil { + return nil, errors.Wrapf(err, "could not resolve excluded ASN %s", target) + } + for _, cidr := range cidrs { + denyList = append(denyList, cidr.String()) + } + continue + } + if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { + parsed, err := urlutil.Parse(target) + if err == nil && parsed.Host != "" { + // URL exclusions apply to the target host, matching the list + // provider's behavior without accidentally matching a query + // string or path. + target = "^" + regexp.QuoteMeta(parsed.Host) + "$" + } + } + denyList = append(denyList, target) + } + if len(denyList) == 0 { + return nil, nil + } + return networkpolicy.New(networkpolicy.Options{DenyList: denyList}) +} + +func targetIsExcluded(target string, policy *networkpolicy.NetworkPolicy) bool { + if policy == nil { + return false + } + parsed, err := urlutil.Parse(target) + if err != nil || parsed.Host == "" { + return !policy.Validate(target) + } + return !policy.Validate(parsed.Host) || !policy.Validate(parsed.Hostname()) +} diff --git a/pkg/input/provider/http/multiformat_target_test.go b/pkg/input/provider/http/multiformat_target_test.go new file mode 100644 index 0000000000..26125ab46b --- /dev/null +++ b/pkg/input/provider/http/multiformat_target_test.go @@ -0,0 +1,156 @@ +package http + +import ( + "path/filepath" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/stretchr/testify/require" +) + +func TestHTTPInputProviderParsesTargetJSONL(t *testing.T) { + const input = `{"url":"https://one.example","tags":["apache"],"severity":["high"]} +{"url":"https://two.example"} +` + + provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputContents: input, + Options: formats.InputFormatOptions{}, + }) + require.NoError(t, err) + require.EqualValues(t, 2, provider.Count()) + require.Equal(t, "TargetInputProvider", provider.InputType()) + + var inputs []*contextargs.MetaInput + provider.Iterate(func(input *contextargs.MetaInput) bool { + inputs = append(inputs, input) + return true + }) + require.Len(t, inputs, 2) + require.Equal(t, "https://one.example", inputs[0].Input) + require.Equal(t, []string{"apache"}, inputs[0].TargetFilter.Tags) + require.Equal(t, "https://two.example", inputs[1].Input) + require.NotNil(t, inputs[1].TargetFilter) + + inputs[0].TargetFilter.Tags[0] = "mutated" + var secondPass []*contextargs.MetaInput + provider.Iterate(func(input *contextargs.MetaInput) bool { + secondPass = append(secondPass, input) + return true + }) + require.Equal(t, []string{"apache"}, secondPass[0].TargetFilter.Tags) +} + +func TestHTTPInputProviderPreservesLegacyJSONL(t *testing.T) { + provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputFile: filepath.Join("..", "..", "formats", "testdata", "ginandjuice.proxify.json"), + Options: formats.InputFormatOptions{}, + }) + require.NoError(t, err) + require.Equal(t, "MultiFormatInputProvider", provider.InputType()) + require.EqualValues(t, 26, provider.Count()) + + count := 0 + provider.Iterate(func(input *contextargs.MetaInput) bool { + require.NotNil(t, input.ReqResp) + require.Nil(t, input.TargetFilter) + count++ + return true + }) + require.Equal(t, 26, count) +} + +func TestHTTPInputProviderDoesNotBuildTargetExclusionsForLegacyJSONL(t *testing.T) { + provider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputFile: filepath.Join("..", "..", "formats", "testdata", "ginandjuice.proxify.json"), + ExcludeTargets: []string{"["}, + Options: formats.InputFormatOptions{}, + }) + + require.NoError(t, err) + require.Equal(t, "MultiFormatInputProvider", provider.InputType()) + require.EqualValues(t, 26, provider.Count()) +} + +func TestHTTPInputProviderValidatesTargetExclusionsForTargetJSONL(t *testing.T) { + _, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputContents: `{"url":"https://example.com"}`, + ExcludeTargets: []string{"["}, + Options: formats.InputFormatOptions{}, + }) + + require.ErrorContains(t, err, "could not prepare target exclusions") +} + +func TestHTTPInputProviderKeepsSameURLWithDifferentFilters(t *testing.T) { + const input = `{"url":"https://same.example","tags":["apache"]} +{"url":"https://same.example","tags":[]} +` + inputProvider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputContents: input, + Options: formats.InputFormatOptions{}, + }) + require.NoError(t, err) + require.Equal(t, "TargetInputProvider", inputProvider.InputType()) + + var ids []string + inputProvider.Iterate(func(input *contextargs.MetaInput) bool { + ids = append(ids, input.ID()) + return true + }) + require.Len(t, ids, 2) + require.NotEqual(t, ids[0], ids[1]) +} + +func TestHTTPInputProviderAppliesTargetExclusions(t *testing.T) { + const input = `{"url":"https://sensitive.example/path","tags":["apache"]} +{"url":"https://192.0.2.25","tags":["apache"]} +{"url":"https://198.51.100.42","tags":["apache"]} +{"url":"https://url-excluded.example/other-path","tags":["apache"]} +{"url":"https://allowed.example/?next=sensitive.example","tags":["apache"]} +{"url":"https://allowed.example","tags":["apache"]} +` + inputProvider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputContents: input, + ExcludeTargets: []string{ + `^sensitive\.example$`, + "192.0.2.25", + "198.51.100.0/24", + "https://url-excluded.example/specific-path", + }, + Options: formats.InputFormatOptions{}, + }) + require.NoError(t, err) + require.Equal(t, "TargetInputProvider", inputProvider.InputType()) + require.EqualValues(t, 2, inputProvider.Count()) + + var inputs []*contextargs.MetaInput + inputProvider.Iterate(func(input *contextargs.MetaInput) bool { + inputs = append(inputs, input) + return true + }) + require.Len(t, inputs, 2) + require.Equal(t, "https://allowed.example/?next=sensitive.example", inputs[0].Input) + require.Equal(t, "https://allowed.example", inputs[1].Input) +} + +func TestHTTPInputProviderRemainsTargetProviderWhenAllTargetsExcluded(t *testing.T) { + const input = `{"url":"https://192.0.2.25","tags":["apache"]}` + inputProvider, err := NewHttpInputProvider(&HttpMultiFormatOptions{ + InputMode: "jsonl", + InputContents: input, + ExcludeTargets: []string{"192.0.2.0/24"}, + Options: formats.InputFormatOptions{}, + }) + + require.NoError(t, err) + require.Equal(t, "TargetInputProvider", inputProvider.InputType()) + require.Zero(t, inputProvider.Count()) +} diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index 33cfbee7fd..ce21620d14 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -28,6 +28,7 @@ var ( const ( MultiFormatInputProvider = "MultiFormatInputProvider" + TargetInputProvider = "TargetInputProvider" ListInputProvider = "ListInputProvider" SimpleListInputProvider = "SimpleInputProvider" ) @@ -74,6 +75,12 @@ type InputProvider interface { Close() } +// PerTargetInputProvider exposes inputs carrying target-specific template +// filters so the runner can resolve inheritance before templates are loaded. +type PerTargetInputProvider interface { + TargetInputs() []*contextargs.MetaInput +} + // InputOptions contains options for input provider type InputOptions struct { // Options for global config @@ -155,8 +162,9 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { } return http.NewHttpInputProvider(&http.HttpMultiFormatOptions{ - InputFile: opts.Options.TargetsFilePath, - InputMode: opts.Options.InputFileMode, + InputFile: opts.Options.TargetsFilePath, + InputMode: opts.Options.InputFileMode, + ExcludeTargets: []string(opts.Options.ExcludeTargets), Options: formats.InputFormatOptions{ Variables: generators.MergeMaps(extraVars, opts.Options.Vars.AsMap()), SkipFormatValidation: opts.Options.SkipFormatValidation, diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go index 9d2f009e81..f357451a87 100644 --- a/pkg/protocols/common/contextargs/metainput.go +++ b/pkg/protocols/common/contextargs/metainput.go @@ -20,6 +20,8 @@ type MetaInput struct { Input string `json:"input,omitempty"` // CustomIP to use for connection CustomIP string `json:"customIP,omitempty"` + // TargetFilter contains optional per-target template selection criteria. + TargetFilter *TargetFilter `json:"target-filter,omitempty"` // hash of the input hash string `json:"-"` @@ -108,13 +110,17 @@ func (metaInput *MetaInput) Address() string { // ID returns a unique id/hash for metainput func (metaInput *MetaInput) ID() string { + filterIdentity := metaInput.TargetFilter.identity() if metaInput.CustomIP != "" { - return fmt.Sprintf("%s-%s", metaInput.Input, metaInput.CustomIP) + if filterIdentity == "" { + return fmt.Sprintf("%s-%s", metaInput.Input, metaInput.CustomIP) + } + return fmt.Sprintf("%s-%s-%s", metaInput.Input, metaInput.CustomIP, filterIdentity) } if metaInput.ReqResp != nil { return metaInput.ReqResp.ID() } - return metaInput.Input + return metaInput.Input + filterIdentity } func (metaInput *MetaInput) MarshalString() (string, error) { @@ -148,6 +154,7 @@ func (metaInput *MetaInput) Clone() *MetaInput { input := NewMetaInput() input.Input = metaInput.Input input.CustomIP = metaInput.CustomIP + input.TargetFilter = metaInput.TargetFilter.clone() input.hash = metaInput.hash if metaInput.ReqResp != nil { input.ReqResp = metaInput.ReqResp.Clone() @@ -178,7 +185,11 @@ func (metaInput *MetaInput) GetScanHash(templateId string) string { if metaInput.ReqResp != nil { rawRequest = metaInput.ReqResp.ID() } - metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP + rawRequest) + filterIdentity := metaInput.TargetFilter.identity() + if filterIdentity != "" { + filterIdentity = ":" + filterIdentity + } + metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP + filterIdentity + rawRequest) } return metaInput.hash } diff --git a/pkg/protocols/common/contextargs/metainput_test.go b/pkg/protocols/common/contextargs/metainput_test.go index 2c60c1a98a..9be97e88ee 100644 --- a/pkg/protocols/common/contextargs/metainput_test.go +++ b/pkg/protocols/common/contextargs/metainput_test.go @@ -3,6 +3,8 @@ package contextargs import ( "testing" + inputtypes "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) @@ -20,3 +22,53 @@ func TestMetaInputMarshalAndUnmarshalString(t *testing.T) { require.Equal(t, input.Input, decoded.Input) require.Equal(t, input.CustomIP, decoded.CustomIP) } + +func TestMetaInputLegacyIdentityRemainsStableWithoutTargetFilter(t *testing.T) { + plain := NewMetaInput() + plain.Input = "https://plain.example" + require.Equal(t, plain.Input, plain.ID()) + require.Equal( + t, + getMd5Hash("template-id:https://plain.example:"), + plain.GetScanHash("template-id"), + ) + + input := NewMetaInput() + input.Input = "https://example.com" + input.CustomIP = "192.0.2.10" + + require.Equal(t, "https://example.com-192.0.2.10", input.ID()) + require.Equal( + t, + getMd5Hash("template-id:https://example.com:192.0.2.10"), + input.GetScanHash("template-id"), + ) + + requestURL, err := urlutil.ParseAbsoluteURL("https://raw.example/request", false) + require.NoError(t, err) + raw := &inputtypes.RequestResponse{URL: *requestURL} + requestInput := NewMetaInput() + requestInput.Input = requestURL.String() + requestInput.ReqResp = raw + require.Equal(t, raw.ID(), requestInput.ID()) + require.Equal( + t, + getMd5Hash("template-id:"+requestInput.Input+":"+raw.ID()), + requestInput.GetScanHash("template-id"), + ) +} + +func TestMetaInputTargetFilterChangesIdentity(t *testing.T) { + first := NewMetaInput() + first.Input = "https://example.com" + first.TargetFilter = &TargetFilter{HasTags: true, Tags: []string{"apache"}} + first.TargetFilter.Prepare(first.TargetFilter.Tags, nil, nil, nil, nil, false) + + second := NewMetaInput() + second.Input = first.Input + second.TargetFilter = &TargetFilter{HasTags: true, Tags: []string{"nginx"}} + second.TargetFilter.Prepare(second.TargetFilter.Tags, nil, nil, nil, nil, false) + + require.NotEqual(t, first.ID(), second.ID()) + require.NotEqual(t, first.GetScanHash("template-id"), second.GetScanHash("template-id")) +} diff --git a/pkg/protocols/common/contextargs/target_filter.go b/pkg/protocols/common/contextargs/target_filter.go new file mode 100644 index 0000000000..49e672ffbc --- /dev/null +++ b/pkg/protocols/common/contextargs/target_filter.go @@ -0,0 +1,212 @@ +package contextargs + +import ( + "path/filepath" + "sort" + "strings" + + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" +) + +// TargetFilter contains optional template selection overrides attached to a +// single input target. The Has* fields distinguish an omitted JSONL field +// (inherit the corresponding global option) from an explicitly provided field. +type TargetFilter struct { + Tags []string `json:"tags,omitempty"` + ExcludeTags []string `json:"exclude-tags,omitempty"` + Severities severity.Severities `json:"severity,omitempty"` + Templates []string `json:"templates,omitempty"` + + HasTags bool `json:"-"` + HasExcludeTags bool `json:"-"` + HasSeverities bool `json:"-"` + HasTemplates bool `json:"-"` + SourceLine int `json:"-"` + + prepared *preparedTargetFilter +} + +// preparedTargetFilter is immutable after construction. MetaInput clones can +// therefore share it without copying potentially large, expanded template +// path sets for every template/target pair. +type preparedTargetFilter struct { + effectiveTags map[string]struct{} + effectiveExcludeTags map[string]struct{} + effectiveSeverities map[severity.Severity]struct{} + effectiveTemplates []string + effectiveIncludes []string + restrictTemplates bool + identity string +} + +// Prepare resolves inheritance and compiles the effective criteria used during +// execution. Callers must finish preparing filters before a scan starts. +func (f *TargetFilter) Prepare(tags, excludeTags []string, severities severity.Severities, templatePaths, includeTemplatePaths []string, restrictTemplates bool) { + if f == nil { + return + } + + prepared := &preparedTargetFilter{ + effectiveTags: stringSet(tags), + effectiveExcludeTags: stringSet(excludeTags), + effectiveSeverities: make(map[severity.Severity]struct{}, len(severities)), + effectiveTemplates: sortedCleanPaths(templatePaths), + effectiveIncludes: sortedCleanPaths(includeTemplatePaths), + restrictTemplates: restrictTemplates, + } + for _, value := range severities { + prepared.effectiveSeverities[value] = struct{}{} + } + prepared.identity = f.buildIdentity() + f.prepared = prepared +} + +// MatchesTemplate reports whether a template is enabled for this target. +// Workflows bypass this matcher to preserve existing workflow semantics. The +// runner rejects workflows when any JSONL target override is present because +// workflow child templates cannot yet apply independent target filters. +func (f *TargetFilter) MatchesTemplate(templatePath string, templateTags []string, templateSeverity severity.Severity, isWorkflow bool) bool { + if f == nil || f.prepared == nil { + return true + } + if isWorkflow { + return true + } + + templatePath = filepath.Clean(templatePath) + if containsSortedPath(f.prepared.effectiveIncludes, templatePath) { + return true + } + + if f.prepared.restrictTemplates { + if !containsSortedPath(f.prepared.effectiveTemplates, templatePath) { + return false + } + } + + if len(f.prepared.effectiveTags) > 0 { + matched := false + for _, tag := range templateTags { + if _, ok := f.prepared.effectiveTags[strings.ToLower(tag)]; ok { + matched = true + break + } + } + if !matched { + return false + } + } + + for _, tag := range templateTags { + if _, excluded := f.prepared.effectiveExcludeTags[strings.ToLower(tag)]; excluded { + return false + } + } + + if len(f.prepared.effectiveSeverities) > 0 { + if _, ok := f.prepared.effectiveSeverities[templateSeverity]; !ok { + return false + } + } + return true +} + +func (f *TargetFilter) clone() *TargetFilter { + if f == nil { + return nil + } + cloned := &TargetFilter{ + Tags: append([]string(nil), f.Tags...), + ExcludeTags: append([]string(nil), f.ExcludeTags...), + Severities: append(severity.Severities(nil), f.Severities...), + Templates: append([]string(nil), f.Templates...), + HasTags: f.HasTags, + HasExcludeTags: f.HasExcludeTags, + HasSeverities: f.HasSeverities, + HasTemplates: f.HasTemplates, + SourceLine: f.SourceLine, + prepared: f.prepared, + } + return cloned +} + +func (f *TargetFilter) identity() string { + if f == nil { + return "" + } + if f.prepared != nil { + return f.prepared.identity + } + return f.buildIdentity() +} + +func (f *TargetFilter) buildIdentity() string { + type canonicalIdentity struct { + Tags []string + ExcludeTags []string + Severities []string + Templates []string + HasTags bool + HasExcludeTags bool + HasSeverities bool + HasTemplates bool + } + identity := canonicalIdentity{ + Tags: sortedStrings(f.Tags), + ExcludeTags: sortedStrings(f.ExcludeTags), + Severities: sortedSeverities(f.Severities), + Templates: sortedStrings(f.Templates), + HasTags: f.HasTags, + HasExcludeTags: f.HasExcludeTags, + HasSeverities: f.HasSeverities, + HasTemplates: f.HasTemplates, + } + data, err := json.Marshal(identity) + if err != nil { + return "" + } + return string(data) +} + +func sortedStrings(values []string) []string { + result := append([]string(nil), values...) + sort.Strings(result) + return result +} + +func sortedSeverities(values severity.Severities) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + result = append(result, value.String()) + } + sort.Strings(result) + return result +} + +// sortedCleanPaths normalizes a defensive copy so containsSortedPath can +// binary-search regardless of caller ordering or path form. +func sortedCleanPaths(paths []string) []string { + if len(paths) == 0 { + return nil + } + result := make([]string, 0, len(paths)) + for _, path := range paths { + result = append(result, filepath.Clean(path)) + } + sort.Strings(result) + return result +} + +func containsSortedPath(paths []string, path string) bool { + index := sort.SearchStrings(paths, path) + return index < len(paths) && paths[index] == path +} + +func stringSet(values []string) map[string]struct{} { + result := make(map[string]struct{}, len(values)) + for _, value := range values { + result[strings.ToLower(value)] = struct{}{} + } + return result +} diff --git a/pkg/protocols/common/contextargs/target_filter_test.go b/pkg/protocols/common/contextargs/target_filter_test.go new file mode 100644 index 0000000000..11cc20dce3 --- /dev/null +++ b/pkg/protocols/common/contextargs/target_filter_test.go @@ -0,0 +1,162 @@ +package contextargs + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/stretchr/testify/require" +) + +func TestTargetFilterMatchesTemplate(t *testing.T) { + filter := &TargetFilter{} + filter.Prepare( + []string{"apache"}, + []string{"deprecated"}, + severity.Severities{severity.High}, + []string{"/templates/apache.yaml"}, + []string{"/templates/forced.yaml"}, + true, + ) + + tests := []struct { + name string + path string + tags []string + severity severity.Severity + isWorkflow bool + expected bool + }{ + { + name: "matching template", + path: "/templates/apache.yaml", + tags: []string{"apache"}, + severity: severity.High, + expected: true, + }, + { + name: "template path rejected", + path: "/templates/other.yaml", + tags: []string{"apache"}, + severity: severity.High, + expected: false, + }, + { + name: "required tag missing", + path: "/templates/apache.yaml", + tags: []string{"nginx"}, + severity: severity.High, + expected: false, + }, + { + name: "excluded tag wins", + path: "/templates/apache.yaml", + tags: []string{"apache", "deprecated"}, + severity: severity.High, + expected: false, + }, + { + name: "severity rejected", + path: "/templates/apache.yaml", + tags: []string{"apache"}, + severity: severity.Low, + expected: false, + }, + { + name: "undefined severity rejected", + path: "/templates/apache.yaml", + tags: []string{"apache"}, + severity: severity.Undefined, + expected: false, + }, + { + name: "explicit include bypasses filters", + path: "/templates/forced.yaml", + tags: []string{"deprecated"}, + severity: severity.Low, + expected: true, + }, + { + name: "workflow bypasses template filters", + path: "/workflows/example.yaml", + tags: nil, + severity: severity.Undefined, + isWorkflow: true, + expected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.expected, filter.MatchesTemplate(test.path, test.tags, test.severity, test.isWorkflow)) + }) + } +} + +func TestTargetFilterPrepareNormalizesTemplatePaths(t *testing.T) { + filter := &TargetFilter{} + filter.Prepare( + []string{"apache"}, + []string{"deprecated"}, + severity.Severities{severity.High}, + []string{"/templates/zzz.yaml", "/templates/aaa.yaml", "/templates/dir/../mmm.yaml"}, + []string{"/forced/b.yaml", "/forced/a.yaml"}, + true, + ) + + // Prepare must store a sorted, filepath.Cleaned copy of both slices so the + // binary search in MatchesTemplate stays correct regardless of caller input. + require.Equal(t, []string{"/templates/aaa.yaml", "/templates/mmm.yaml", "/templates/zzz.yaml"}, filter.prepared.effectiveTemplates) + require.Equal(t, []string{"/forced/a.yaml", "/forced/b.yaml"}, filter.prepared.effectiveIncludes) + + // A template restricted-in only through normalization still matches once the + // tag and severity criteria are satisfied. + require.True(t, filter.MatchesTemplate("/templates/mmm.yaml", []string{"apache"}, severity.High, false)) + + // Forced includes bypass the restrictive tag, exclude-tag, and severity + // criteria that would otherwise reject these templates. + for _, path := range []string{"/forced/a.yaml", "/forced/b.yaml"} { + require.True(t, filter.MatchesTemplate(path, []string{"deprecated", "nginx"}, severity.Low, false), path) + } + + // A template outside both sets is still rejected under restrictTemplates. + require.False(t, filter.MatchesTemplate("/templates/other.yaml", []string{"apache"}, severity.High, false)) +} + +func TestTargetFilterIdentityIsCanonicalAndPresenceAware(t *testing.T) { + omitted := &TargetFilter{} + explicitEmpty := &TargetFilter{HasTags: true, Tags: []string{}} + require.NotEqual(t, omitted.identity(), explicitEmpty.identity()) + + first := &TargetFilter{ + HasTags: true, + HasSeverities: true, + Tags: []string{"shiro", "apache"}, + Severities: severity.Severities{severity.High, severity.Critical}, + } + second := &TargetFilter{ + HasTags: true, + HasSeverities: true, + Tags: []string{"apache", "shiro"}, + Severities: severity.Severities{severity.Critical, severity.High}, + } + first.Prepare(first.Tags, nil, first.Severities, nil, nil, false) + second.Prepare(second.Tags, nil, second.Severities, nil, nil, false) + require.Equal(t, first.identity(), second.identity()) +} + +func TestTargetFilterCloneSharesOnlyImmutablePreparedState(t *testing.T) { + original := &TargetFilter{ + HasTags: true, + Tags: []string{"apache"}, + } + original.Prepare(original.Tags, []string{"deprecated"}, severity.Severities{severity.High}, []string{"/one.yaml"}, []string{"/forced.yaml"}, true) + + cloned := original.clone() + cloned.Tags[0] = "nginx" + + require.Equal(t, []string{"apache"}, original.Tags) + require.Same(t, original.prepared, cloned.prepared) + require.True(t, cloned.MatchesTemplate("/one.yaml", []string{"apache"}, severity.High, false)) + require.False(t, cloned.MatchesTemplate("/one.yaml", []string{"nginx"}, severity.High, false)) + require.Equal(t, original.identity(), cloned.identity()) +} diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 9dfc3f7bb4..c3ada27b95 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -246,10 +247,30 @@ func (e *ClusterExecuter) Requests() int { return count } +// MatchesTargetFilter reports whether at least one original template in this +// cluster is enabled for the target. The shared request is skipped entirely +// when every clustered operator is filtered out. +func (e *ClusterExecuter) MatchesTargetFilter(filter *contextargs.TargetFilter) bool { + if filter == nil { + return true + } + for _, operator := range e.operators { + if operatorMatchesTargetFilter(operator, filter) { + return true + } + } + return false +} + // Execute executes the protocol group and returns true or false if results were found. func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { var results bool + filter := ctx.Input.MetaInput.TargetFilter + if !e.MatchesTargetFilter(filter) { + return false, nil + } + inputItem := ctx.Input.Clone() if e.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" { if inputItem.MetaInput.Input = e.options.InputHelper.Transform(ctx.Input.MetaInput.Input, e.templateType); ctx.Input.MetaInput.Input == "" { @@ -271,6 +292,9 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { event.InternalEvent = make(map[string]interface{}) } for _, operator := range e.operators { + if !operatorMatchesTargetFilter(operator, filter) { + continue + } clonedEvent := event.CloneShallow() result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) @@ -298,6 +322,9 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // Parse URL fields from the input fields := protocolUtils.GetJsonFieldsFromURL(ctx.Input.MetaInput.Input) for _, operator := range e.operators { + if !operatorMatchesTargetFilter(operator, filter) { + continue + } errMsg := "" if err != nil { errMsg = err.Error() @@ -339,6 +366,11 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R scanCtx := scan.NewScanContext(ctx.Context(), ctx.Input) dynamicValues := make(map[string]interface{}) + filter := ctx.Input.MetaInput.TargetFilter + if !e.MatchesTargetFilter(filter) { + return nil, nil + } + inputItem := ctx.Input.Clone() if e.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" { if inputItem.MetaInput.Input = e.options.InputHelper.Transform(ctx.Input.MetaInput.Input, e.templateType); ctx.Input.MetaInput.Input == "" { @@ -347,6 +379,9 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } err := e.requests.ExecuteWithResults(inputItem, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { + if !operatorMatchesTargetFilter(operator, filter) { + continue + } clonedEvent := event.CloneShallow() result, matched := operator.operator.Execute(clonedEvent.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) @@ -369,3 +404,15 @@ func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.R } return scanCtx.GenerateResult(), err } + +func operatorMatchesTargetFilter(operator *clusteredOperator, filter *contextargs.TargetFilter) bool { + if filter == nil { + return true + } + return filter.MatchesTemplate( + operator.templatePath, + operator.templateInfo.Tags.ToSlice(), + operator.templateInfo.SeverityHolder.Severity, + false, + ) +} diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index e40a4c42ec..8180de7fa1 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -3,11 +3,14 @@ package templates import ( "testing" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v3/pkg/operators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/stretchr/testify/require" ) @@ -50,3 +53,44 @@ func TestClusterTemplates(t *testing.T) { require.ElementsMatch(t, got, expected) }) } + +func TestClusterExecuterMatchesPerTargetFilters(t *testing.T) { + cluster := &ClusterExecuter{ + operators: []*clusteredOperator{ + { + templateID: "apache-template", + templatePath: "/templates/apache.yaml", + templateInfo: model.Info{ + Tags: stringslice.New([]string{"apache"}), + SeverityHolder: severity.Holder{Severity: severity.High}, + }, + operator: &operators.Operators{}, + }, + { + templateID: "nginx-template", + templatePath: "/templates/nginx.yaml", + templateInfo: model.Info{ + Tags: stringslice.New([]string{"nginx"}), + SeverityHolder: severity.Holder{Severity: severity.Medium}, + }, + operator: &operators.Operators{}, + }, + }, + } + + apache := &contextargs.TargetFilter{} + apache.Prepare([]string{"apache"}, nil, nil, nil, nil, false) + require.True(t, cluster.MatchesTargetFilter(apache)) + require.True(t, operatorMatchesTargetFilter(cluster.operators[0], apache)) + require.False(t, operatorMatchesTargetFilter(cluster.operators[1], apache)) + + none := &contextargs.TargetFilter{} + none.Prepare([]string{"tomcat"}, nil, nil, nil, nil, false) + require.False(t, cluster.MatchesTargetFilter(none)) + + all := &contextargs.TargetFilter{} + all.Prepare(nil, nil, nil, nil, nil, false) + require.True(t, cluster.MatchesTargetFilter(all)) + require.True(t, operatorMatchesTargetFilter(cluster.operators[0], all)) + require.True(t, operatorMatchesTargetFilter(cluster.operators[1], all)) +}