diff --git a/internal/runner/options_test.go b/internal/runner/options_test.go index 92a6fc7662..9ac014453e 100644 --- a/internal/runner/options_test.go +++ b/internal/runner/options_test.go @@ -49,7 +49,7 @@ func TestParseHeadlessOptionalArguments(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { strsl := goflags.StringSlice{} - for _, v := range strings.Split(tt.input, ",") { + for v := range strings.SplitSeq(tt.input, ",") { //nolint strsl.Set(v) } diff --git a/internal/runner/preflight_portscan.go b/internal/runner/preflight_portscan.go index 1f3bd1a9f2..26691882ef 100644 --- a/internal/runner/preflight_portscan.go +++ b/internal/runner/preflight_portscan.go @@ -543,9 +543,8 @@ func splitPorts(s string) []string { if s == "" { return nil } - parts := strings.Split(s, ",") - out := make([]string, 0, len(parts)) - for _, p := range parts { + var out []string + for p := range strings.SplitSeq(s, ",") { p = strings.TrimSpace(p) if p != "" { out = append(out, p) diff --git a/internal/tests/integration/http_test.go b/internal/tests/integration/http_test.go index 6002395998..0ee817830a 100644 --- a/internal/tests/integration/http_test.go +++ b/internal/tests/integration/http_test.go @@ -829,7 +829,7 @@ func (h *httpRawUnsafePath) Execute(filepath string) error { } actual := []string{} - for _, v := range strings.Split(results, "\n") { + for v := range strings.SplitSeq(results, "\n") { if strings.Contains(v, "GET") { parts := strings.Fields(v) if len(parts) == 3 { @@ -877,7 +877,7 @@ func (h *httpPaths) Execute(filepath string) error { } actual := []string{} - for _, v := range strings.Split(results, "\n") { + for v := range strings.SplitSeq(results, "\n") { if strings.Contains(v, "GET") { parts := strings.Fields(v) if len(parts) == 3 { @@ -1394,7 +1394,7 @@ func (h *httpVariableDSLFunction) Execute(filePath string) error { } actual := []string{} - for _, v := range strings.Split(results, "\n") { + for v := range strings.SplitSeq(results, "\n") { if strings.Contains(v, "GET") { parts := strings.Fields(v) if len(parts) == 3 { @@ -1763,7 +1763,7 @@ func (h *httpRawPathSingleSlash) Execute(filepath string) error { } var actual string - for _, v := range strings.Split(results, "\n") { + for v := range strings.SplitSeq(results, "\n") { if strings.Contains(v, "GET") { parts := strings.Fields(v) if len(parts) == 3 { @@ -1788,7 +1788,7 @@ func (h *httpRawUnsafePathSingleSlash) Execute(filepath string) error { } var actual string - for _, v := range strings.Split(results, "\n") { + for v := range strings.SplitSeq(results, "\n") { if strings.Contains(v, "GET") { parts := strings.Fields(v) if len(parts) == 3 { diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index eeff4b1c97..dc0feffc9e 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -18,11 +18,16 @@ var knownLeaks = []goleak.Option{ // net/http transport maintains idle keep-alive connections whose goroutines // exit on idle timeout or explicit close - not real leaks. goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"), + goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"), + goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"), // expirable LRU cache creates a background goroutine for TTL expiration that persists // see: https://github.com/hashicorp/golang-lru/blob/770151e9c8cdfae1797826b7b74c33d6f103fbd8/expirable/expirable_lru.go#L79 goleak.IgnoreAnyContainingPkg("github.com/hashicorp/golang-lru/v2/expirable"), - goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"), - goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"), + // httpcache leveldb + shared rate limiter + memguardian outlive a single SDK engine. + // Sleep alone is not reliable under -race or when NewNucleiEngineCtx fails before Close. + goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"), + goleak.IgnoreAnyContainingPkg("github.com/projectdiscovery/ratelimit"), + goleak.IgnoreAnyFunction("github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate.StartActiveMemGuardian.func1"), } func TestSimpleNuclei(t *testing.T) { @@ -38,12 +43,12 @@ func TestSimpleNuclei(t *testing.T) { nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // filter dns templates nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), ) - require.Nil(t, err) + require.NoError(t, err) + defer ne.Close() ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here // when callback is nil it nuclei will print JSON output to stdout err = ne.ExecuteWithCallback(nil) - require.Nil(t, err) - defer ne.Close() + require.NoError(t, err) } // this is shared test so needs to be run as separate process @@ -76,14 +81,14 @@ func TestSimpleNucleiRemote(t *testing.T) { }, ), ) - require.Nil(t, err) + require.NoError(t, err) + defer ne.Close() ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here err = ne.LoadAllTemplates() - require.Nil(t, err, "could not load templates") + require.NoError(t, err, "could not load templates") // when callback is nil it nuclei will print JSON output to stdout err = ne.ExecuteWithCallback(nil) - require.Nil(t, err) - defer ne.Close() + require.NoError(t, err) } // this is shared test so needs to be run as separate process if env.GetEnvOrDefault("TestSimpleNucleiRemote", false) { @@ -108,22 +113,20 @@ func TestThreadSafeNuclei(t *testing.T) { }() // create nuclei engine with options ne, err := nuclei.NewThreadSafeNucleiEngineCtx(context.TODO()) - require.Nil(t, err) + require.NoError(t, err) + defer ne.Close() // scan 1 = run dns templates on scanme.sh t.Run("scanme.sh", func(t *testing.T) { err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) - require.Nil(t, err) + require.NoError(t, err) }) // scan 2 = run dns templates on honey.scanme.sh t.Run("honey.scanme.sh", func(t *testing.T) { err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) - require.Nil(t, err) + require.NoError(t, err) }) - - // wait for all scans to finish - defer ne.Close() } if env.GetEnvOrDefault("TestThreadSafeNuclei", false) { @@ -153,11 +156,11 @@ func TestWithVarsNuclei(t *testing.T) { nuclei.WithVars([]string{"token=foobar"}), nuclei.WithVerbosity(nuclei.VerbosityOptions{Debug: true}), ) - require.Nil(t, err) + require.NoError(t, err) + defer ne.Close() ne.LoadTargets([]string{"scanme.sh"}, true) // probe http/https target is set to true here err = ne.ExecuteWithCallback(nil) - require.Nil(t, err) - defer ne.Close() + require.NoError(t, err) } // this is shared test so needs to be run as separate process if env.GetEnvOrDefault("TestWithVarsNuclei", false) { diff --git a/pkg/catalog/config/nucleiconfig.go b/pkg/catalog/config/nucleiconfig.go index d271170192..a1ddf69832 100644 --- a/pkg/catalog/config/nucleiconfig.go +++ b/pkg/catalog/config/nucleiconfig.go @@ -356,17 +356,12 @@ func (c *Config) IsDebugArgEnabled(arg string) bool { // parseDebugArgs from string func (c *Config) parseDebugArgs(data string) { // use space as separator instead of commas - tmp := strings.Fields(data) - for _, v := range tmp { + for v := range strings.FieldsSeq(data) { key := v value := "" // if it is key value pair then split it - if strings.Contains(v, "=") { - parts := strings.SplitN(v, "=", 2) - if len(parts) != 2 { - continue - } - key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + if k, val, ok := strings.Cut(v, "="); ok { + key, value = strings.TrimSpace(k), strings.TrimSpace(val) } if value == "false" || value == "0" { // if false or disabled then skip diff --git a/pkg/catalog/config/template.go b/pkg/catalog/config/template.go index 3c0a43f4dc..fa90df080c 100644 --- a/pkg/catalog/config/template.go +++ b/pkg/catalog/config/template.go @@ -101,8 +101,7 @@ func IsTemplateWithRoot(fpath, rootDir string) bool { // Only check components if pathToCheck is NOT absolute // This avoids false positives on parent directories for absolute paths if !filepath.IsAbs(pathToCheck) { - parts := strings.Split(pathToCheck, string(os.PathSeparator)) - for _, p := range parts { + for p := range strings.SplitSeq(pathToCheck, string(os.PathSeparator)) { for _, excluded := range knownMiscDirectories { if strings.EqualFold(p, excluded) { return false diff --git a/pkg/installer/template.go b/pkg/installer/template.go index ad7c109c50..78223927e8 100644 --- a/pkg/installer/template.go +++ b/pkg/installer/template.go @@ -92,6 +92,10 @@ func (t *TemplateManager) FreshInstallIfNotExists() error { // UpdateIfOutdated updates templates if they are outdated func (t *TemplateManager) UpdateIfOutdated() error { + return withTemplatesUpdateLock(t.updateIfOutdatedLocked) +} + +func (t *TemplateManager) updateIfOutdatedLocked() error { // if the templates folder does not exist, it's a fresh installation and do not update if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { return t.FreshInstallIfNotExists() @@ -583,8 +587,10 @@ func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, err checksums, err := os.ReadFile(checksumFilePath) if err == nil { allChecksums := make(map[string]string) - for _, v := range strings.Split(string(checksums), ";") { + checksumStr := string(checksums) + for v := range strings.SplitSeq(checksumStr, ";") { v = strings.TrimSpace(v) + // Strict two-field parse: paths may contain commas (Cut would parse wrong). tmparr := strings.Split(v, ",") if len(tmparr) != 2 { continue diff --git a/pkg/installer/update_lock.go b/pkg/installer/update_lock.go new file mode 100644 index 0000000000..abbcffa87c --- /dev/null +++ b/pkg/installer/update_lock.go @@ -0,0 +1,42 @@ +package installer + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/projectdiscovery/utils/errkit" +) + +const templatesUpdateLockName = "nuclei-templates-update.lock" + +// withTemplatesUpdateLock serializes template install/update across processes. +// Parallel `go test` packages each have their own sync.Once, so without a +// cross-process lock they race on the shared templates directory (ENOENT, +// partial trees, checksum mismatches). +func withTemplatesUpdateLock(fn func() error) error { + lockPath := filepath.Join(os.TempDir(), templatesUpdateLockName) + deadline := time.Now().Add(10 * time.Minute) + + for { + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err == nil { + _, _ = fmt.Fprintf(f, "%d\n", os.Getpid()) + defer func() { + _ = f.Close() + _ = os.Remove(lockPath) + }() + return fn() + } + + if info, statErr := os.Stat(lockPath); statErr == nil && time.Since(info.ModTime()) > 15*time.Minute { + _ = os.Remove(lockPath) + continue + } + if time.Now().After(deadline) { + return errkit.Wrap(err, "timed out waiting for nuclei templates update lock") + } + time.Sleep(250 * time.Millisecond) + } +} diff --git a/pkg/js/devtools/tsgen/parser.go b/pkg/js/devtools/tsgen/parser.go index a615b4739d..6dc2d9aa36 100644 --- a/pkg/js/devtools/tsgen/parser.go +++ b/pkg/js/devtools/tsgen/parser.go @@ -572,8 +572,7 @@ func updateFuncWithConstructorSig(sig string, f Function) Function { sig = strings.TrimPrefix(sig, "constructor(") sig = strings.TrimSuffix(sig, ")") // split by comma - args := strings.Split(sig, ",") - for _, arg := range args { + for arg := range strings.SplitSeq(sig, ",") { arg = strings.TrimSpace(arg) // check if it is optional typeData := strings.Split(arg, ":") diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index d91d9358fa..53eaa3796a 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -874,7 +874,7 @@ func (request *Request) getPorts() []string { if strings.EqualFold(k, "Port") { portStr := types.ToString(v) ports := []string{} - for _, p := range strings.Split(portStr, ",") { + for p := range strings.SplitSeq(portStr, ",") { trimmed := strings.TrimSpace(p) if trimmed != "" { ports = append(ports, trimmed) diff --git a/pkg/protocols/network/network.go b/pkg/protocols/network/network.go index 695de654d6..4d27acef25 100644 --- a/pkg/protocols/network/network.go +++ b/pkg/protocols/network/network.go @@ -207,7 +207,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { // parse ports and validate if request.Port != "" { seen := make(map[string]struct{}) - for _, port := range strings.Split(request.Port, ",") { + for port := range strings.SplitSeq(request.Port, ",") { port = strings.TrimSpace(port) if port == "" { continue diff --git a/pkg/templates/fuzz_harness.go b/pkg/templates/fuzz_harness.go index 1ff84c2a2c..6c1d59a8a4 100644 --- a/pkg/templates/fuzz_harness.go +++ b/pkg/templates/fuzz_harness.go @@ -234,7 +234,8 @@ func (candidate *fuzzTemplateCandidate) yaml() []byte { builder.WriteString("http:\n - ") if candidate.useRawRequest { builder.WriteString("raw:\n - |\n") - for _, line := range strings.Split(candidate.rawRequest(), "\r\n") { + rawReq := candidate.rawRequest() + for line := range strings.SplitSeq(rawReq, "\r\n") { if line == "" { builder.WriteString(" \n") continue diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 41e9d0d6d0..dbe596a129 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -1,6 +1,7 @@ package flow import ( + "bufio" "fmt" "io" "strconv" @@ -331,14 +332,19 @@ func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { defer func() { _ = reader.Close() }() - bin, err := io.ReadAll(reader) - if err != nil { - return values, err - } - for _, line := range strings.Split(string(bin), "\n") { - line = strings.TrimSpace(line) - if line != "" { - values = append(values, line) + // Stream the helper file line-by-line instead of loading it whole into memory: + // helper files for flow templates can be large (wordlists, payload corpora). + br := bufio.NewReader(reader) + for { + line, readErr := br.ReadString('\n') + if trimmed := strings.TrimSpace(line); trimmed != "" { + values = append(values, trimmed) + } + if readErr == io.EOF { + break + } + if readErr != nil { + return values, readErr } } return values, nil