diff --git a/internal/runner/options.go b/internal/runner/options.go index 8b3cfcd541..71247652e4 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -2,6 +2,7 @@ package runner import ( "bufio" + "bytes" "fmt" "io/fs" "os" @@ -298,6 +299,17 @@ func validateDASTOptions(options *types.Options) error { return nil } +// LoadReportingOptionsFromBytes parses YAML reporting-config bytes into a +// *reporting.Options with env-var expansion, matching the CLI's -report-config. +func LoadReportingOptionsFromBytes(data []byte) (*reporting.Options, error) { + reportingOptions := &reporting.Options{} + if err := yaml.DecodeAndValidate(bytes.NewReader(data), reportingOptions); err != nil { + return nil, errors.Wrap(err, "could not parse reporting config file") + } + Walk(reportingOptions, expandEndVars) + return reportingOptions, nil +} + func createReportingOptions(options *types.Options) (*reporting.Options, error) { var reportingOptions = &reporting.Options{} if options.ReportingConfig != "" { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 33f20f27ba..1d69112303 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -51,10 +51,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" diff --git a/lib/config.go b/lib/config.go index b19e99faf1..795f058dd8 100644 --- a/lib/config.go +++ b/lib/config.go @@ -1,6 +1,7 @@ package nuclei import ( + "bytes" "context" "errors" "os" @@ -8,8 +9,10 @@ import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/internal/runner" "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/utils/errkit" + "gopkg.in/yaml.v2" "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" @@ -556,6 +559,173 @@ func WithOptions(opts *pkgtypes.Options) NucleiSDKOptions { } } +// WithPDCPUpload uploads findings to the PDCP dashboard, matching the CLI's +// `-dashboard -scan-id -team-id`. Credentials come from PDCP_API_KEY or +// ~/.config/nuclei/.pdcp/credentials.yaml; missing creds log a warning and +// scans continue. A non-empty scanID implicitly enables upload. +func WithPDCPUpload(scanID, teamID string) NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.EnableCloudUpload = true + if scanID != "" { + e.opts.ScanID = scanID + } + if teamID != "" { + e.opts.TeamID = teamID + } + return nil + } +} + +// RuntimeConfig is the set of nuclei config options exposed via the SDK. +type RuntimeConfig struct { + Authors []string `yaml:"author,omitempty"` + Tags []string `yaml:"tags,omitempty"` + ExcludeTags []string `yaml:"exclude-tags,omitempty"` + IncludeTags []string `yaml:"include-tags,omitempty"` + IncludeIds []string `yaml:"template-id,omitempty"` + ExcludeIds []string `yaml:"exclude-id,omitempty"` + IncludeTemplates []string `yaml:"include-templates,omitempty"` + ExcludedTemplates []string `yaml:"exclude-templates,omitempty"` + ExcludeMatchers []string `yaml:"exclude-matchers,omitempty"` + Severities []string `yaml:"severity,omitempty"` + ExcludeSeverities []string `yaml:"exclude-severity,omitempty"` + Protocols []string `yaml:"type,omitempty"` + ExcludeProtocols []string `yaml:"exclude-type,omitempty"` + IncludeConditions []string `yaml:"template-condition,omitempty"` + Headers []string `yaml:"header,omitempty"` + Variables []string `yaml:"var,omitempty"` + InteractshServer string `yaml:"interactsh-server,omitempty"` + InteractshToken string `yaml:"interactsh-token,omitempty"` + Socks5Proxy []string `yaml:"socks5-proxy,omitempty"` + // Scalar knobs use *int so omitted YAML keys preserve the engine's + // existing value instead of forcing it to zero. + RateLimit *int `yaml:"rate-limit,omitempty"` + BulkSize *int `yaml:"bulk-size,omitempty"` + Concurrency *int `yaml:"concurrency,omitempty"` // maps to opts.TemplateThreads + Timeout *int `yaml:"timeout,omitempty"` + Retries *int `yaml:"retries,omitempty"` + RateLimitHost *int `yaml:"rate-limit-host,omitempty"` +} + +// MergeOptions appends/sets the configuration onto opts. +// +// RateLimitHost is stored on the struct for downstream consumers but is NOT +// applied to *types.Options — there is no equivalent field on nuclei's +// runtime options today. Callers needing per-host rate limiting must wire it +// outside the engine. +func (s *RuntimeConfig) MergeOptions(opts *pkgtypes.Options) { + opts.Authors = append(opts.Authors, s.Authors...) + opts.Tags = append(opts.Tags, s.Tags...) + opts.ExcludeTags = append(opts.ExcludeTags, s.ExcludeTags...) + opts.IncludeTags = append(opts.IncludeTags, s.IncludeTags...) + opts.IncludeIds = append(opts.IncludeIds, s.IncludeIds...) + opts.ExcludeIds = append(opts.ExcludeIds, s.ExcludeIds...) + opts.IncludeTemplates = append(opts.IncludeTemplates, s.IncludeTemplates...) + opts.ExcludedTemplates = append(opts.ExcludedTemplates, s.ExcludedTemplates...) + opts.ExcludeMatchers = append(opts.ExcludeMatchers, s.ExcludeMatchers...) + opts.IncludeConditions = append(opts.IncludeConditions, s.IncludeConditions...) + if s.InteractshServer != "" { + opts.InteractshURL = s.InteractshServer + } + if s.InteractshToken != "" { + opts.InteractshToken = s.InteractshToken + } + for _, v := range s.Severities { + _ = opts.Severities.Set(v) + } + for _, v := range s.ExcludeSeverities { + _ = opts.ExcludeSeverities.Set(v) + } + for _, v := range s.Protocols { + _ = opts.Protocols.Set(v) + } + for _, v := range s.ExcludeProtocols { + _ = opts.ExcludeProtocols.Set(v) + } + for _, v := range s.Headers { + opts.CustomHeaders = append(opts.CustomHeaders, v) + } + for _, v := range s.Variables { + _ = opts.Vars.Set(v) + } + opts.Proxy = append(opts.Proxy, s.Socks5Proxy...) + + if s.RateLimit != nil { + opts.RateLimit = *s.RateLimit + } + if s.BulkSize != nil { + opts.BulkSize = *s.BulkSize + } + if s.Concurrency != nil { + opts.TemplateThreads = *s.Concurrency + } + if s.Timeout != nil { + opts.Timeout = *s.Timeout + } + if s.Retries != nil { + opts.Retries = *s.Retries + } +} + +// WithConfigFile decodes a RuntimeConfig YAML at path and merges it into +// the engine options. Matches the schema Aurora server emits. +func WithConfigFile(path string) NucleiSDKOptions { + return func(e *NucleiEngine) error { + data, err := os.ReadFile(path) + if err != nil { + return errkit.Wrap(err, "could not open nuclei config file") + } + return applyRuntimeConfigFromBytes(e, data) + } +} + +// WithConfigBytes is WithConfigFile from memory. +func WithConfigBytes(data []byte) NucleiSDKOptions { + return func(e *NucleiEngine) error { + return applyRuntimeConfigFromBytes(e, data) + } +} + +func applyRuntimeConfigFromBytes(e *NucleiEngine, data []byte) error { + cfg := &RuntimeConfig{} + if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(cfg); err != nil { + return errkit.Wrap(err, "could not parse nuclei config") + } + cfg.MergeOptions(e.opts) + return nil +} + +// WithReportingConfigFile loads a nuclei -report-config style YAML file +// (Jira/Linear/GitHub/etc. tracker configuration) into the engine's +// reporting options. Equivalent to -report-config on the CLI. +func WithReportingConfigFile(path string) NucleiSDKOptions { + return func(e *NucleiEngine) error { + data, err := os.ReadFile(path) + if err != nil { + return errkit.Wrap(err, "could not open reporting config file") + } + ropts, err := runner.LoadReportingOptionsFromBytes(data) + if err != nil { + return errkit.Wrap(err, "could not parse reporting config file") + } + e.reportingOpts = ropts + return nil + } +} + +// WithReportingConfigBytes is WithReportingConfigFile from memory. Passing +// nil/empty produces an empty reporting.Options (no-op). +func WithReportingConfigBytes(data []byte) NucleiSDKOptions { + return func(e *NucleiEngine) error { + ropts, err := runner.LoadReportingOptionsFromBytes(data) + if err != nil { + return errkit.Wrap(err, "could not parse reporting config bytes") + } + e.reportingOpts = ropts + return nil + } +} + // WithTemporaryDirectory allows setting a parent directory for SDK-managed temporary files. // A temporary directory will be created inside the provided directory and cleaned up on engine close. // If not set, a temporary directory will be automatically created in the system temp location. diff --git a/lib/config_test.go b/lib/config_test.go new file mode 100644 index 0000000000..fda14d9642 --- /dev/null +++ b/lib/config_test.go @@ -0,0 +1,116 @@ +package nuclei + +import ( + "os" + "path/filepath" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" + "github.com/stretchr/testify/require" +) + +func TestWithConfigFile(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "nuclei.yaml") + cfg := `tags: + - cve +severity: + - high + - critical +exclude-tags: + - dos +header: + - "X-Test: 1" +` + require.NoError(t, os.WriteFile(cfgPath, []byte(cfg), 0o600)) + + ne, err := NewNucleiEngine(WithConfigFile(cfgPath)) + require.NoError(t, err) + defer ne.Close() + + opts := ne.Options() + require.Contains(t, opts.Tags, "cve") + require.Contains(t, opts.ExcludeTags, "dos") + require.Contains(t, opts.CustomHeaders, "X-Test: 1") + + got := map[severity.Severity]bool{} + for _, s := range opts.Severities { + got[s] = true + } + require.True(t, got[severity.High]) + require.True(t, got[severity.Critical]) +} + +func TestWithConfigBytes(t *testing.T) { + cfg := []byte("tags:\n - cve\ntemplate-id:\n - CVE-2024-0001\n") + + ne, err := NewNucleiEngine(WithConfigBytes(cfg)) + require.NoError(t, err) + defer ne.Close() + + opts := ne.Options() + require.Contains(t, opts.Tags, "cve") + require.Contains(t, opts.IncludeIds, "CVE-2024-0001") +} + +func TestWithConfigBytes_ScalarKnobs(t *testing.T) { + cfg := []byte("rate-limit: 99\nbulk-size: 7\nconcurrency: 42\ntimeout: 30\nretries: 5\n") + + ne, err := NewNucleiEngine(WithConfigBytes(cfg)) + require.NoError(t, err) + defer ne.Close() + + opts := ne.Options() + require.Equal(t, 99, opts.RateLimit) + require.Equal(t, 7, opts.BulkSize) + require.Equal(t, 42, opts.TemplateThreads) + require.Equal(t, 30, opts.Timeout) + require.Equal(t, 5, opts.Retries) +} + +func TestWithReportingConfigFile(t *testing.T) { + dir := t.TempDir() + rcPath := filepath.Join(dir, "report.yaml") + rc := `github: + username: test-user + owner: test-owner + token: test-token + project-name: test-project + issue-label: test +` + require.NoError(t, os.WriteFile(rcPath, []byte(rc), 0o600)) + + ne, err := NewNucleiEngine(WithReportingConfigFile(rcPath)) + require.NoError(t, err) + defer ne.Close() + + ropts := ne.reportingOptionsForTest() + require.NotNil(t, ropts) + require.NotNil(t, ropts.GitHub) + require.Equal(t, "test-user", ropts.GitHub.Username) + require.Equal(t, "test-owner", ropts.GitHub.Owner) +} + +func TestWithReportingConfigBytes(t *testing.T) { + rc := []byte(`github: + username: test-user + owner: test-owner + token: test-token + project-name: test-project +`) + ne, err := NewNucleiEngine(WithReportingConfigBytes(rc)) + require.NoError(t, err) + defer ne.Close() + + ropts := ne.reportingOptionsForTest() + require.NotNil(t, ropts) + require.NotNil(t, ropts.GitHub) + require.Equal(t, "test-user", ropts.GitHub.Username) +} + +// Invalid YAML must return an error, not a silently empty config. +func TestWithReportingConfigBytes_InvalidYAML(t *testing.T) { + rc := []byte("this: is: not: valid: yaml: ::::\n") + _, err := NewNucleiEngine(WithReportingConfigBytes(rc)) + require.Error(t, err) +} diff --git a/lib/example_test.go b/lib/example_test.go index 81c7fc106e..f7bd494605 100644 --- a/lib/example_test.go +++ b/lib/example_test.go @@ -75,6 +75,41 @@ func ExampleThreadSafeNucleiEngine() { // [caa-fingerprint] honey.scanme.sh } +// ExampleWithPDCPUpload uploads findings to the PDCP dashboard from SDK code, +// matching `-dashboard -scan-id -team-id` on the CLI. Pass an existing scanID +// to append; pass empty to let the server create a new scan. +func ExampleWithPDCPUpload() { + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"self-signed-ssl"}}), + nuclei.WithPDCPUpload("" /* scanID */, "" /* teamID, "" = personal */), + ) + if err != nil { + panic(err) + } + defer ne.Close() + ne.LoadTargets([]string{"scanme.sh"}, false) + if err := ne.ExecuteWithCallback(nil); err != nil { + panic(err) + } +} + +// ExampleWithConfigFile ingests a RuntimeConfig YAML (tags, severity, +// exclude-tags, headers, vars, etc.) and merges it into the engine options. +func ExampleWithConfigFile() { + ne, err := nuclei.NewNucleiEngine( + nuclei.WithConfigFile("nuclei.yaml"), + nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"self-signed-ssl"}}), + ) + if err != nil { + panic(err) + } + defer ne.Close() + ne.LoadTargets([]string{"scanme.sh"}, false) + if err := ne.ExecuteWithCallback(nil); err != nil { + panic(err) + } +} + func TestMain(m *testing.M) { // this file only contains testtables examples https://go.dev/blog/examples // and actual functionality test are in sdk_test.go diff --git a/lib/internal_test.go b/lib/internal_test.go new file mode 100644 index 0000000000..9596056a45 --- /dev/null +++ b/lib/internal_test.go @@ -0,0 +1,9 @@ +package nuclei + +import "github.com/projectdiscovery/nuclei/v3/pkg/reporting" + +// reportingOptionsForTest exposes e.reportingOpts to same-package tests. +// In a _test.go file so it stays off the public SDK surface. +func (e *NucleiEngine) reportingOptionsForTest() *reporting.Options { + return e.reportingOpts +} diff --git a/lib/sdk.go b/lib/sdk.go index 4f1740b452..8c2d8435b0 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -89,6 +89,7 @@ type NucleiEngine struct { customWriter output.Writer customProgress progress.Progress rc reporting.Client + reportingOpts *reporting.Options executerOpts *protocols.ExecutorOptions // Logger instance for the engine diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 9f427dd0ba..a1c7db3c31 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -15,7 +15,9 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v3/internal/pdcp" "github.com/projectdiscovery/nuclei/v3/internal/runner" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" @@ -32,10 +34,10 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/types" nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/ratelimit" + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" ) // applyRequiredDefaults to options @@ -73,6 +75,34 @@ func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) { e.customWriter = mockoutput } + // Inline PDCP upload wiring; mirrors the CLI's setupPDCPUpload without + // pulling it into the exported runner surface. + if e.opts.ScanID != "" { + e.opts.EnableCloudUpload = true + } + if e.opts.EnableCloudUpload { + creds, err := (&pdcpauth.PDCPCredHandler{}).GetCreds() + if err != nil { + if err != pdcpauth.ErrNoCreds { + e.Logger.Verbose().Msgf("Could not get credentials for cloud upload: %s\n", err) + } + e.Logger.Warning().Msgf("To view results on Cloud Dashboard, configure API key from %v", pdcpauth.DashBoardURL) + } else if uploadWriter, err := pdcp.NewUploadWriter(ctx, e.Logger, creds); err != nil { + e.Logger.Warning().Msgf("PDCP (%v) Auto-Save Failed: %s", pdcpauth.DashBoardURL, err) + } else { + if e.opts.ScanID != "" { + _ = uploadWriter.SetScanID(e.opts.ScanID) + } + if e.opts.ScanName != "" { + uploadWriter.SetScanName(e.opts.ScanName) + } + if e.opts.TeamID != "" { + uploadWriter.SetTeamID(e.opts.TeamID) + } + e.customWriter = output.NewMultiWriter(e.customWriter, uploadWriter) + } + } + if e.customProgress == nil { e.customProgress = &testutils.MockProgressClient{} } @@ -170,8 +200,11 @@ func (e *NucleiEngine) init(ctx context.Context) error { if err := reporting.CreateConfigIfNotExists(); err != nil { return err } - // we don't support reporting config in sdk mode - if e.rc, err = reporting.New(&reporting.Options{}, "", false); err != nil { + ropts := e.reportingOpts + if ropts == nil { + ropts = &reporting.Options{} + } + if e.rc, err = reporting.New(ropts, "", false); err != nil { return err } e.interactshOpts.IssuesClient = e.rc