From c51aac6c9c4bc2048b712cdc54c2283ec40bbf30 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Tue, 15 Jul 2025 03:04:08 -0500 Subject: [PATCH 01/13] support for concurrent nuclei engines --- lib/config.go | 2 +- lib/sdk_private.go | 19 ++++ pkg/installer/template.go | 2 +- pkg/protocols/common/protocolstate/state.go | 4 +- pkg/protocols/dns/dnsclientpool/clientpool.go | 17 ++- .../http/httpclientpool/clientpool.go | 12 +-- pkg/protocols/protocols.go | 58 ++++++++-- .../whois/rdapclientpool/clientpool.go | 10 +- pkg/templates/compile.go | 101 ++++++++++++------ pkg/templates/parser.go | 98 +++-------------- 10 files changed, 187 insertions(+), 136 deletions(-) diff --git a/lib/config.go b/lib/config.go index 5e96352b57..125442898d 100644 --- a/lib/config.go +++ b/lib/config.go @@ -537,7 +537,7 @@ func WithResumeFile(file string) NucleiSDKOptions { } } -// WithLogger allows setting gologger instance +// WithLogger allows setting a shared gologger instance func WithLogger(logger *gologger.Logger) NucleiSDKOptions { return func(e *NucleiEngine) error { e.Logger = logger diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 659187b200..d80a0fd068 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -231,6 +231,25 @@ func (e *NucleiEngine) init(ctx context.Context) error { } } + // Handle the case where the user passed an existing parser that we can use as a cache + if e.opts.Parser != nil { + if cachedParser, ok := e.opts.Parser.(*templates.Parser); ok { + e.parser = cachedParser + e.opts.Parser = cachedParser + e.executerOpts.Parser = cachedParser + e.executerOpts.Options.Parser = cachedParser + } + } + + // Create a new parser if necessary + if e.parser == nil { + op := templates.NewParser() + e.parser = op + e.opts.Parser = op + e.executerOpts.Parser = op + e.executerOpts.Options.Parser = op + } + e.engine = core.New(e.opts) e.engine.SetExecuterOptions(e.executerOpts) diff --git a/pkg/installer/template.go b/pkg/installer/template.go index 4ee784477e..9e56f12a18 100644 --- a/pkg/installer/template.go +++ b/pkg/installer/template.go @@ -53,7 +53,7 @@ func (t *templateUpdateResults) String() string { }, } table := tablewriter.NewWriter(&buff) - table.Header("Total", "Added", "Modified", "Removed") + table.Header([]string{"Total", "Added", "Modified", "Removed"}) for _, v := range data { _ = table.Append(v) } diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 9f9a96a067..3abe9310c0 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -201,8 +201,8 @@ func initDialers(options *types.Options) error { StartActiveMemGuardian(context.Background()) // TODO: this should be tied to executionID - // overidde global settings with latest options - LfaAllowed = options.AllowLocalFileAccess + // override global settings with latest options + // LfaAllowed = options.AllowLocalFileAccess return nil } diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go index c1805be1cb..040dd007f6 100644 --- a/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -11,9 +11,11 @@ import ( ) var ( - poolMutex *sync.RWMutex + poolMutex *sync.RWMutex + clientPool map[string]*retryabledns.Client + normalClient *retryabledns.Client - clientPool map[string]*retryabledns.Client + m sync.Mutex ) // defaultResolvers contains the list of resolvers known to be trusted. @@ -26,6 +28,9 @@ var defaultResolvers = []string{ // Init initializes the client pool implementation func Init(options *types.Options) error { + m.Lock() + defer m.Unlock() + // Don't create clients if already created in the past. if normalClient != nil { return nil @@ -45,6 +50,12 @@ func Init(options *types.Options) error { return nil } +func getNormalClient() *retryabledns.Client { + m.Lock() + defer m.Unlock() + return normalClient +} + // Configuration contains the custom configuration options for a client type Configuration struct { // Retries contains the retries for the dns client @@ -71,7 +82,7 @@ func (c *Configuration) Hash() string { // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 { - return normalClient, nil + return getNormalClient(), nil } hash := configuration.Hash() poolMutex.RLock() diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 940ac38866..4fa0790a54 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -154,16 +154,16 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { return dialers.RawHTTPClient } - rawHttpOptions := rawhttp.DefaultOptions + rawHttpOptionsCopy := *rawhttp.DefaultOptions if options.Options.AliveHttpProxy != "" { - rawHttpOptions.Proxy = options.Options.AliveHttpProxy + rawHttpOptionsCopy.Proxy = options.Options.AliveHttpProxy } else if options.Options.AliveSocksProxy != "" { - rawHttpOptions.Proxy = options.Options.AliveSocksProxy + rawHttpOptionsCopy.Proxy = options.Options.AliveSocksProxy } else if dialers.Fastdialer != nil { - rawHttpOptions.FastDialer = dialers.Fastdialer + rawHttpOptionsCopy.FastDialer = dialers.Fastdialer } - rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout - dialers.RawHTTPClient = rawhttp.NewClient(rawHttpOptions) + rawHttpOptionsCopy.Timeout = options.Options.GetTimeouts().HttpTimeout + dialers.RawHTTPClient = rawhttp.NewClient(&rawHttpOptionsCopy) return dialers.RawHTTPClient } diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 30443eee63..4d208a3275 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -198,6 +198,11 @@ func (e *ExecutorOptions) HasTemplateCtx(input *contextargs.MetaInput) bool { // GetTemplateCtx returns template context for given input func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context { scanId := input.GetScanHash(e.TemplateID) + if e.templateCtxStore == nil { + // if template context store is not initialized create it + e.CreateTemplateCtxStore() + } + // get template context from store templateCtx, ok := e.templateCtxStore.Get(scanId) if !ok { // if template context does not exist create new and add it to store and return it @@ -444,14 +449,49 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) { if e == nil || n == nil || n.Options == nil { return } - execID := n.Options.GetExecutionID() - e.SetExecutionID(execID) -} -// ApplyNewEngineOptions updates an existing ExecutorOptions with options from a new engine. This -// handles things like the ExecutionID that need to be updated. -func (e *ExecutorOptions) SetExecutionID(executorId string) { - e.m.Lock() - defer e.m.Unlock() - e.Options.SetExecutionID(executorId) + // The types.Options include the ExecutionID among other things + e.Options = n.Options.Copy() + + // Keep the template-specific fields, but replace the rest + /* + e.TemplateID = n.TemplateID + e.TemplatePath = n.TemplatePath + e.TemplateInfo = n.TemplateInfo + e.TemplateVerifier = n.TemplateVerifier + e.RawTemplate = n.RawTemplate + e.Variables = n.Variables + e.Constants = n.Constants + */ + e.Output = n.Output + e.Options = n.Options + e.IssuesClient = n.IssuesClient + e.Progress = n.Progress + e.RateLimiter = n.RateLimiter + e.Catalog = n.Catalog + e.ProjectFile = n.ProjectFile + e.Browser = n.Browser + e.Interactsh = n.Interactsh + e.HostErrorsCache = n.HostErrorsCache + e.StopAtFirstMatch = n.StopAtFirstMatch + e.ExcludeMatchers = n.ExcludeMatchers + e.InputHelper = n.InputHelper + e.FuzzParamsFrequency = n.FuzzParamsFrequency + e.FuzzStatsDB = n.FuzzStatsDB + e.DoNotCache = n.DoNotCache + e.Colorizer = n.Colorizer + e.WorkflowLoader = n.WorkflowLoader + e.ResumeCfg = n.ResumeCfg + e.ProtocolType = n.ProtocolType + e.Flow = n.Flow + e.IsMultiProtocol = n.IsMultiProtocol + e.templateCtxStore = n.templateCtxStore + e.JsCompiler = n.JsCompiler + e.AuthProvider = n.AuthProvider + e.TemporaryDirectory = n.TemporaryDirectory + e.Parser = n.Parser + e.ExportReqURLPattern = n.ExportReqURLPattern + e.GlobalMatchers = n.GlobalMatchers + e.Logger = n.Logger + e.CustomFastdialer = n.CustomFastdialer } diff --git a/pkg/protocols/whois/rdapclientpool/clientpool.go b/pkg/protocols/whois/rdapclientpool/clientpool.go index 81da1c578c..f2d4f2316c 100644 --- a/pkg/protocols/whois/rdapclientpool/clientpool.go +++ b/pkg/protocols/whois/rdapclientpool/clientpool.go @@ -30,6 +30,12 @@ func Init(options *types.Options) error { return nil } +func getNormalClient() *rdap.Client { + m.Lock() + defer m.Unlock() + return normalClient +} + // Configuration contains the custom configuration options for a client - placeholder type Configuration struct{} @@ -40,7 +46,5 @@ func (c *Configuration) Hash() string { // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) { - m.Lock() - defer m.Unlock() - return normalClient, nil + return getNormalClient(), nil } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index a0d99a7686..fdb612a96c 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -56,49 +56,90 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut } if !options.DoNotCache { if value, _, _ := parser.compiledTemplatesCache.Has(filePath); value != nil { - // Update the template to use the current options for the calling engine - // TODO: This may be require additional work for robustness - t := *value - t.Options.ApplyNewEngineOptions(options) - if t.CompiledWorkflow != nil { - t.CompiledWorkflow.Options.ApplyNewEngineOptions(options) - for _, w := range t.CompiledWorkflow.Workflows { + // Copy the template, apply new options, and recompile requests + tplCopy := *value + newBase := options.Copy() + newBase.TemplateID = tplCopy.Options.TemplateID + newBase.TemplatePath = tplCopy.Options.TemplatePath + newBase.TemplateInfo = tplCopy.Options.TemplateInfo + newBase.TemplateVerifier = tplCopy.Options.TemplateVerifier + newBase.RawTemplate = tplCopy.Options.RawTemplate + tplCopy.Options = newBase + + tplCopy.Options.ApplyNewEngineOptions(options) + if tplCopy.CompiledWorkflow != nil { + tplCopy.CompiledWorkflow.Options.ApplyNewEngineOptions(options) + for _, w := range tplCopy.CompiledWorkflow.Workflows { for _, ex := range w.Executers { ex.Options.ApplyNewEngineOptions(options) } } } - for _, r := range t.RequestsDNS { - r.UpdateOptions(t.Options) + + // TODO: Reconsider whether to recompile requests. Compiling these is just as slow + // as not using a cache at all, but may be necessary. + + for i, r := range tplCopy.RequestsDNS { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsDNS[i] = &rCopy } - for _, r := range t.RequestsHTTP { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsHTTP { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsHTTP[i] = &rCopy } - for _, r := range t.RequestsCode { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsCode { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsCode[i] = &rCopy } - for _, r := range t.RequestsFile { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsFile { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsFile[i] = &rCopy } - for _, r := range t.RequestsHeadless { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsHeadless { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsHeadless[i] = &rCopy } - for _, r := range t.RequestsNetwork { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsNetwork { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsNetwork[i] = &rCopy } - for _, r := range t.RequestsJavascript { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsJavascript { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + //rCopy.Compile(tplCopy.Options) + tplCopy.RequestsJavascript[i] = &rCopy } - for _, r := range t.RequestsSSL { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsSSL { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsSSL[i] = &rCopy } - for _, r := range t.RequestsWHOIS { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsWHOIS { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsWHOIS[i] = &rCopy } - for _, r := range t.RequestsWebsocket { - r.UpdateOptions(t.Options) + for i, r := range tplCopy.RequestsWebsocket { + rCopy := *r + rCopy.UpdateOptions(tplCopy.Options) + // rCopy.Compile(tplCopy.Options) + tplCopy.RequestsWebsocket[i] = &rCopy } - template := t + template := &tplCopy if template.isGlobalMatchersEnabled() { item := &globalmatchers.Item{ @@ -119,8 +160,8 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut template.CompiledWorkflow = compiled template.CompiledWorkflow.Options = options } - - return &template, nil + // options.Logger.Error().Msgf("returning cached template %s after recompiling %d requests", tplCopy.Options.TemplateID, tplCopy.Requests()) + return template, nil } } diff --git a/pkg/templates/parser.go b/pkg/templates/parser.go index b995299161..6aae4053af 100644 --- a/pkg/templates/parser.go +++ b/pkg/templates/parser.go @@ -49,6 +49,23 @@ func (p *Parser) Cache() *Cache { return p.parsedTemplatesCache } +// Cache returns the parsed templates cache +func (p *Parser) CompiledCache() *Cache { + return p.compiledTemplatesCache +} + +func (p *Parser) ParsedCount() int { + p.Lock() + defer p.Unlock() + return len(p.parsedTemplatesCache.items.Map) +} + +func (p *Parser) CompiledCount() int { + p.Lock() + defer p.Unlock() + return len(p.compiledTemplatesCache.items.Map) +} + func checkOpenFileError(err error) bool { if err != nil && strings.Contains(err.Error(), "too many open files") { panic(err) @@ -171,84 +188,3 @@ func (p *Parser) LoadWorkflow(templatePath string, catalog catalog.Catalog) (boo return false, nil } - -// CloneForExecutionId creates a clone with updated execution IDs -func (p *Parser) CloneForExecutionId(xid string) *Parser { - p.Lock() - defer p.Unlock() - - newParser := &Parser{ - ShouldValidate: p.ShouldValidate, - NoStrictSyntax: p.NoStrictSyntax, - parsedTemplatesCache: NewCache(), - compiledTemplatesCache: NewCache(), - } - - for k, tpl := range p.parsedTemplatesCache.items.Map { - newTemplate := templateUpdateExecutionId(tpl.template, xid) - newParser.parsedTemplatesCache.Store(k, newTemplate, []byte(tpl.raw), tpl.err) - } - - for k, tpl := range p.compiledTemplatesCache.items.Map { - newTemplate := templateUpdateExecutionId(tpl.template, xid) - newParser.compiledTemplatesCache.Store(k, newTemplate, []byte(tpl.raw), tpl.err) - } - - return newParser -} - -func templateUpdateExecutionId(tpl *Template, xid string) *Template { - // TODO: This is a no-op today since options are patched in elsewhere, but we're keeping this - // for future work where we may need additional tweaks per template instance. - return tpl - - /* - templateBase := *tpl - var newOpts *protocols.ExecutorOptions - // Swap out the types.Options execution ID attached to the template - if templateBase.Options != nil { - optionsBase := *templateBase.Options //nolint - templateBase.Options = &optionsBase - if templateBase.Options.Options != nil { - optionsOptionsBase := *templateBase.Options.Options //nolint - templateBase.Options.Options = &optionsOptionsBase - templateBase.Options.Options.ExecutionId = xid - newOpts = templateBase.Options - } - } - if newOpts == nil { - return &templateBase - } - for _, r := range templateBase.RequestsDNS { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsHTTP { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsCode { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsFile { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsHeadless { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsNetwork { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsJavascript { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsSSL { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsWHOIS { - r.UpdateOptions(newOpts) - } - for _, r := range templateBase.RequestsWebsocket { - r.UpdateOptions(newOpts) - } - return &templateBase - */ -} From 82e29c956e09be930314207e8a462614c0da3807 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Tue, 15 Jul 2025 03:07:15 -0500 Subject: [PATCH 02/13] clarify LfaAllowed race --- pkg/protocols/common/protocolstate/state.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 3abe9310c0..285a139a68 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -200,9 +200,8 @@ func initDialers(options *types.Options) error { StartActiveMemGuardian(context.Background()) - // TODO: this should be tied to executionID - // override global settings with latest options - // LfaAllowed = options.AllowLocalFileAccess + // TODO: this is a race and should also be tied to executionID + LfaAllowed = options.AllowLocalFileAccess return nil } From 9ce79aed86fbbcc8484277e0354213a91ce42997 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Tue, 15 Jul 2025 03:20:38 -0500 Subject: [PATCH 03/13] remove unused mutex --- pkg/protocols/protocols.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 4d208a3275..197d79e0a9 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -3,7 +3,6 @@ package protocols import ( "context" "encoding/base64" - "sync" "sync/atomic" "github.com/projectdiscovery/fastdialer/fastdialer" @@ -139,8 +138,6 @@ type ExecutorOptions struct { Logger *gologger.Logger // CustomFastdialer is a fastdialer dialer instance CustomFastdialer *fastdialer.Dialer - - m sync.Mutex } // todo: centralizing components is not feasible with current clogged architecture From b94462df99e92c476102a9500f9813c8317109e2 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Wed, 16 Jul 2025 12:53:51 -0500 Subject: [PATCH 04/13] update LfaAllowed logic to prevent races until it can be reworked for per-execution ID --- pkg/protocols/common/protocolstate/file.go | 46 ++++++++++++++++++- .../common/protocolstate/headless.go | 12 ----- pkg/protocols/common/protocolstate/state.go | 3 +- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pkg/protocols/common/protocolstate/file.go b/pkg/protocols/common/protocolstate/file.go index 9475aac0fc..c4f04293ca 100644 --- a/pkg/protocols/common/protocolstate/file.go +++ b/pkg/protocols/common/protocolstate/file.go @@ -2,8 +2,10 @@ package protocolstate import ( "strings" + "sync" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -11,15 +13,55 @@ import ( var ( // LfaAllowed means local file access is allowed LfaAllowed bool + lfaMutex sync.Mutex ) +// IsLfaAllowed returns whether local file access is allowed +func IsLfaAllowed(options *types.Options) bool { + // Use the global when no options are provided + if options == nil { + lfaMutex.Lock() + defer lfaMutex.Unlock() + return LfaAllowed + } + // Otherwise the specific options + dialers, ok := dialers.Get(options.ExecutionId) + if ok && dialers != nil { + dialers.Lock() + defer dialers.Unlock() + + return dialers.LocalFileAccessAllowed + } + return false +} + +func SetLfaAllowed(options *types.Options) { + // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. + lfaMutex.Lock() + if options != nil { + LfaAllowed = options.AllowLocalFileAccess + } + lfaMutex.Unlock() +} + +func GetLfaAllowed(options *types.Options) bool { + if options != nil { + return options.AllowLocalFileAccess + } + // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. + lfaMutex.Lock() + defer lfaMutex.Unlock() + return LfaAllowed +} + // Normalizepath normalizes path and returns absolute path // it returns error if path is not allowed // this respects the sandbox rules and only loads files from // allowed directories func NormalizePath(filePath string) (string, error) { - // TODO: this should be tied to executionID - if LfaAllowed { + // TODO: this should be tied to executionID using *types.Options + if IsLfaAllowed(nil) { + // if local file access is allowed, we can return the absolute path return filePath, nil } cleaned, err := fileutil.ResolveNClean(filePath, config.DefaultConfig.GetTemplateDir()) diff --git a/pkg/protocols/common/protocolstate/headless.go b/pkg/protocols/common/protocolstate/headless.go index 4012e2da6b..1d9970119c 100644 --- a/pkg/protocols/common/protocolstate/headless.go +++ b/pkg/protocols/common/protocolstate/headless.go @@ -74,18 +74,6 @@ func InitHeadless(options *types.Options) { } } -// AllowLocalFileAccess returns whether local file access is allowed -func IsLfaAllowed(options *types.Options) bool { - dialers, ok := dialers.Get(options.ExecutionId) - if ok && dialers != nil { - dialers.Lock() - defer dialers.Unlock() - - return dialers.LocalFileAccessAllowed - } - return false -} - func IsRestrictLocalNetworkAccess(options *types.Options) bool { dialers, ok := dialers.Get(options.ExecutionId) if ok && dialers != nil { diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 285a139a68..f72122c19c 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -200,8 +200,7 @@ func initDialers(options *types.Options) error { StartActiveMemGuardian(context.Background()) - // TODO: this is a race and should also be tied to executionID - LfaAllowed = options.AllowLocalFileAccess + SetLfaAllowed(options) return nil } From c3fb1d4e64db42a6f5b1ed9ca428f133880d72e3 Mon Sep 17 00:00:00 2001 From: HD Moore Date: Wed, 16 Jul 2025 16:28:21 -0500 Subject: [PATCH 05/13] Update pkg/templates/parser.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/templates/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/templates/parser.go b/pkg/templates/parser.go index 6aae4053af..56b64c237c 100644 --- a/pkg/templates/parser.go +++ b/pkg/templates/parser.go @@ -49,7 +49,7 @@ func (p *Parser) Cache() *Cache { return p.parsedTemplatesCache } -// Cache returns the parsed templates cache +// CompiledCache returns the compiled templates cache func (p *Parser) CompiledCache() *Cache { return p.compiledTemplatesCache } From 6d3c943dd9dadad9346bb6a98be54ffc94e4d972 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 13:41:19 +0200 Subject: [PATCH 06/13] debug tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0c3ab083b4..04fbaee7c8 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ syntax-docs: test: GOFLAGS = -race -v test: - $(GOTEST) $(GOFLAGS) ./... + $(GOTEST) $(GOFLAGS) -failfast -p 1 ./... integration: cd integration_tests; bash run.sh From aa9cc26af5573e5a530c5e8181669bdc2bfd8af1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 14:18:29 +0200 Subject: [PATCH 07/13] debug gh action --- pkg/external/customtemplates/github_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/external/customtemplates/github_test.go b/pkg/external/customtemplates/github_test.go index 972706af18..4b9b502f67 100644 --- a/pkg/external/customtemplates/github_test.go +++ b/pkg/external/customtemplates/github_test.go @@ -2,22 +2,31 @@ package customtemplates import ( "context" + "os" "path/filepath" "testing" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - osutils "github.com/projectdiscovery/utils/os" "github.com/stretchr/testify/require" ) +// stdoutWriter adapts os.Stdout to the gologger writer interface +type stdoutWriter struct{} + +func (w *stdoutWriter) Write(data []byte, level levels.Level) { + os.Stdout.Write(data) +} + func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { - if osutils.IsOSX() { - t.Skip("skipping on macos due to unknown failure (works locally)") - } + // if osutils.IsOSX() { + // t.Skip("skipping on macos due to unknown failure (works locally)") + // } - gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) + gologger.DefaultLogger.SetWriter(&stdoutWriter{}) + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) templatesDirectory := t.TempDir() config.DefaultConfig.SetTemplatesDir(templatesDirectory) @@ -29,5 +38,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { require.Nil(t, err, "could not create custom templates manager") ctm.Download(context.Background()) + require.DirExists(t, filepath.Join(templatesDirectory, "github", "projectdiscovery", "nuclei-templates-test"), "cloned directory does not exists") } From f0dfda6542b17366350c421b9385278ff69606d2 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 15:15:56 +0200 Subject: [PATCH 08/13] fixig gh template test --- pkg/external/customtemplates/github_test.go | 25 ++++++++++----------- pkg/utils/capture_writer.go | 16 +++++++++++++ 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 pkg/utils/capture_writer.go diff --git a/pkg/external/customtemplates/github_test.go b/pkg/external/customtemplates/github_test.go index 4b9b502f67..4e429c0208 100644 --- a/pkg/external/customtemplates/github_test.go +++ b/pkg/external/customtemplates/github_test.go @@ -1,31 +1,24 @@ package customtemplates import ( + "bytes" "context" - "os" "path/filepath" + "strings" "testing" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/stretchr/testify/require" ) -// stdoutWriter adapts os.Stdout to the gologger writer interface -type stdoutWriter struct{} - -func (w *stdoutWriter) Write(data []byte, level levels.Level) { - os.Stdout.Write(data) -} - func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { - // if osutils.IsOSX() { - // t.Skip("skipping on macos due to unknown failure (works locally)") - // } - - gologger.DefaultLogger.SetWriter(&stdoutWriter{}) + // Capture output to check for rate limit errors + outputBuffer := &bytes.Buffer{} + gologger.DefaultLogger.SetWriter(&utils.CaptureWriter{Buffer: outputBuffer}) gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) templatesDirectory := t.TempDir() @@ -39,5 +32,11 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) { ctm.Download(context.Background()) + // Check if output contains rate limit error and skip test if so + output := outputBuffer.String() + if strings.Contains(output, "API rate limit exceeded") { + t.Skip("GitHub API rate limit exceeded, skipping test") + } + require.DirExists(t, filepath.Join(templatesDirectory, "github", "projectdiscovery", "nuclei-templates-test"), "cloned directory does not exists") } diff --git a/pkg/utils/capture_writer.go b/pkg/utils/capture_writer.go new file mode 100644 index 0000000000..29986a5aa9 --- /dev/null +++ b/pkg/utils/capture_writer.go @@ -0,0 +1,16 @@ +package utils + +import ( + "bytes" + + "github.com/projectdiscovery/gologger/levels" +) + +// CaptureWriter captures log output for testing +type CaptureWriter struct { + Buffer *bytes.Buffer +} + +func (w *CaptureWriter) Write(data []byte, level levels.Level) { + w.Buffer.Write(data) +} From ec1559b5a3f644bd2cae36dd31c1b1f6bf36f2e6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 15:44:59 +0200 Subject: [PATCH 09/13] using atomic --- pkg/protocols/common/protocolstate/file.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pkg/protocols/common/protocolstate/file.go b/pkg/protocols/common/protocolstate/file.go index c4f04293ca..d8feb30717 100644 --- a/pkg/protocols/common/protocolstate/file.go +++ b/pkg/protocols/common/protocolstate/file.go @@ -2,7 +2,7 @@ package protocolstate import ( "strings" - "sync" + "sync/atomic" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -12,17 +12,14 @@ import ( var ( // LfaAllowed means local file access is allowed - LfaAllowed bool - lfaMutex sync.Mutex + LfaAllowed atomic.Bool ) // IsLfaAllowed returns whether local file access is allowed func IsLfaAllowed(options *types.Options) bool { // Use the global when no options are provided if options == nil { - lfaMutex.Lock() - defer lfaMutex.Unlock() - return LfaAllowed + return LfaAllowed.Load() } // Otherwise the specific options dialers, ok := dialers.Get(options.ExecutionId) @@ -37,11 +34,9 @@ func IsLfaAllowed(options *types.Options) bool { func SetLfaAllowed(options *types.Options) { // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. - lfaMutex.Lock() if options != nil { - LfaAllowed = options.AllowLocalFileAccess + LfaAllowed.Store(options.AllowLocalFileAccess) } - lfaMutex.Unlock() } func GetLfaAllowed(options *types.Options) bool { @@ -49,9 +44,7 @@ func GetLfaAllowed(options *types.Options) bool { return options.AllowLocalFileAccess } // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. - lfaMutex.Lock() - defer lfaMutex.Unlock() - return LfaAllowed + return LfaAllowed.Load() } // Normalizepath normalizes path and returns absolute path From bb4d952a9f93a3edbda4dbba2e49b4bb5e58959f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 15:45:08 +0200 Subject: [PATCH 10/13] using synclockmap --- pkg/protocols/dns/dnsclientpool/clientpool.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go index 040dd007f6..2f82a139b1 100644 --- a/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -8,11 +8,12 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryabledns" + mapsutil "github.com/projectdiscovery/utils/maps" ) var ( - poolMutex *sync.RWMutex - clientPool map[string]*retryabledns.Client + poolMutex sync.RWMutex + clientPool *mapsutil.SyncLockMap[string, *retryabledns.Client] normalClient *retryabledns.Client m sync.Mutex @@ -35,8 +36,7 @@ func Init(options *types.Options) error { if normalClient != nil { return nil } - poolMutex = &sync.RWMutex{} - clientPool = make(map[string]*retryabledns.Client) + clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]() resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { @@ -85,12 +85,9 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl return getNormalClient(), nil } hash := configuration.Hash() - poolMutex.RLock() - if client, ok := clientPool[hash]; ok { - poolMutex.RUnlock() + if client, ok := clientPool.Get(hash); ok { return client, nil } - poolMutex.RUnlock() resolvers := defaultResolvers if len(options.InternalResolversList) > 0 { @@ -106,9 +103,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl if err != nil { return nil, errors.Wrap(err, "could not create dns client") } + _ = clientPool.Set(hash, client) - poolMutex.Lock() - clientPool[hash] = client - poolMutex.Unlock() return client, nil } From 848f30dd2c942bd025a5526aa5f80735d685a401 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 15:46:29 +0200 Subject: [PATCH 11/13] restore tests concurrency --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 04fbaee7c8..0c3ab083b4 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ syntax-docs: test: GOFLAGS = -race -v test: - $(GOTEST) $(GOFLAGS) -failfast -p 1 ./... + $(GOTEST) $(GOFLAGS) ./... integration: cd integration_tests; bash run.sh From 40f37a9f7113c772e1794df9f48281b5a8f30011 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 15:49:19 +0200 Subject: [PATCH 12/13] lint --- pkg/protocols/dns/dnsclientpool/clientpool.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go index 2f82a139b1..ccbc1bc5d9 100644 --- a/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -12,7 +12,6 @@ import ( ) var ( - poolMutex sync.RWMutex clientPool *mapsutil.SyncLockMap[string, *retryabledns.Client] normalClient *retryabledns.Client From b56b477b71efec75a7f4f81c3239edd0aa26d3cc Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 17:27:34 +0200 Subject: [PATCH 13/13] wiring executionId in js fs --- pkg/js/libs/fs/fs.go | 21 ++++++----- pkg/protocols/common/protocolstate/file.go | 42 +++++++++++++--------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/pkg/js/libs/fs/fs.go b/pkg/js/libs/fs/fs.go index e3a3fd7bdd..a5f77e8757 100644 --- a/pkg/js/libs/fs/fs.go +++ b/pkg/js/libs/fs/fs.go @@ -1,6 +1,7 @@ package fs import ( + "context" "os" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" @@ -27,8 +28,9 @@ import ( // // when no itemType is provided, it will return both files and directories // const items = fs.ListDir('/tmp'); // ``` -func ListDir(path string, itemType string) ([]string, error) { - finalPath, err := protocolstate.NormalizePath(path) +func ListDir(ctx context.Context, path string, itemType string) ([]string, error) { + executionId := ctx.Value("executionId").(string) + finalPath, err := protocolstate.NormalizePathWithExecutionId(executionId, path) if err != nil { return nil, err } @@ -57,8 +59,9 @@ func ListDir(path string, itemType string) ([]string, error) { // // here permitted directories are $HOME/nuclei-templates/* // const content = fs.ReadFile('helpers/usernames.txt'); // ``` -func ReadFile(path string) ([]byte, error) { - finalPath, err := protocolstate.NormalizePath(path) +func ReadFile(ctx context.Context, path string) ([]byte, error) { + executionId := ctx.Value("executionId").(string) + finalPath, err := protocolstate.NormalizePathWithExecutionId(executionId, path) if err != nil { return nil, err } @@ -74,8 +77,8 @@ func ReadFile(path string) ([]byte, error) { // // here permitted directories are $HOME/nuclei-templates/* // const content = fs.ReadFileAsString('helpers/usernames.txt'); // ``` -func ReadFileAsString(path string) (string, error) { - bin, err := ReadFile(path) +func ReadFileAsString(ctx context.Context, path string) (string, error) { + bin, err := ReadFile(ctx, path) if err != nil { return "", err } @@ -91,14 +94,14 @@ func ReadFileAsString(path string) (string, error) { // const contents = fs.ReadFilesFromDir('helpers/ssh-keys'); // log(contents); // ``` -func ReadFilesFromDir(dir string) ([]string, error) { - files, err := ListDir(dir, "file") +func ReadFilesFromDir(ctx context.Context, dir string) ([]string, error) { + files, err := ListDir(ctx, dir, "file") if err != nil { return nil, err } var results []string for _, file := range files { - content, err := ReadFileAsString(dir + "/" + file) + content, err := ReadFileAsString(ctx, dir+"/"+file) if err != nil { return nil, err } diff --git a/pkg/protocols/common/protocolstate/file.go b/pkg/protocols/common/protocolstate/file.go index d8feb30717..180d5a0b5e 100644 --- a/pkg/protocols/common/protocolstate/file.go +++ b/pkg/protocols/common/protocolstate/file.go @@ -2,26 +2,30 @@ package protocolstate import ( "strings" - "sync/atomic" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" ) var ( // LfaAllowed means local file access is allowed - LfaAllowed atomic.Bool + LfaAllowed *mapsutil.SyncLockMap[string, bool] ) +func init() { + LfaAllowed = mapsutil.NewSyncLockMap[string, bool]() +} + // IsLfaAllowed returns whether local file access is allowed func IsLfaAllowed(options *types.Options) bool { - // Use the global when no options are provided - if options == nil { - return LfaAllowed.Load() + if GetLfaAllowed(options) { + return true } - // Otherwise the specific options + + // Otherwise look into dialers dialers, ok := dialers.Get(options.ExecutionId) if ok && dialers != nil { dialers.Lock() @@ -29,31 +33,35 @@ func IsLfaAllowed(options *types.Options) bool { return dialers.LocalFileAccessAllowed } - return false + + // otherwise just return option value + return options.AllowLocalFileAccess } func SetLfaAllowed(options *types.Options) { - // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. - if options != nil { - LfaAllowed.Store(options.AllowLocalFileAccess) - } + _ = LfaAllowed.Set(options.ExecutionId, options.AllowLocalFileAccess) } func GetLfaAllowed(options *types.Options) bool { - if options != nil { - return options.AllowLocalFileAccess + allowed, ok := LfaAllowed.Get(options.ExecutionId) + + return ok && allowed +} + +func NormalizePathWithExecutionId(executionId string, filePath string) (string, error) { + options := &types.Options{ + ExecutionId: executionId, } - // TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks. - return LfaAllowed.Load() + return NormalizePath(options, filePath) } // Normalizepath normalizes path and returns absolute path // it returns error if path is not allowed // this respects the sandbox rules and only loads files from // allowed directories -func NormalizePath(filePath string) (string, error) { +func NormalizePath(options *types.Options, filePath string) (string, error) { // TODO: this should be tied to executionID using *types.Options - if IsLfaAllowed(nil) { + if IsLfaAllowed(options) { // if local file access is allowed, we can return the absolute path return filePath, nil }