Support concurrent Nuclei engines in the same process

lib/config.go

@@ -537,7 +537,7 @@ func WithResumeFile(file string) NucleiSDKOptions {
 	}
 }
 
-// WithLogger allows setting gologger instance
+// WithLogger allows setting a shared gologger instance
 func WithLogger(logger *gologger.Logger) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
 		e.Logger = logger

lib/sdk_private.go

@@ -231,6 +231,25 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		}
 	}
 
+	// Handle the case where the user passed an existing parser that we can use as a cache
+	if e.opts.Parser != nil {
+		if cachedParser, ok := e.opts.Parser.(*templates.Parser); ok {
+			e.parser = cachedParser
+			e.opts.Parser = cachedParser
+			e.executerOpts.Parser = cachedParser
+			e.executerOpts.Options.Parser = cachedParser
+		}
+	}
+
+	// Create a new parser if necessary
+	if e.parser == nil {
+		op := templates.NewParser()
+		e.parser = op
+		e.opts.Parser = op
+		e.executerOpts.Parser = op
+		e.executerOpts.Options.Parser = op
+	}
+
 	e.engine = core.New(e.opts)
 	e.engine.SetExecuterOptions(e.executerOpts)
 

pkg/installer/template.go

@@ -53,7 +53,7 @@ func (t *templateUpdateResults) String() string {
 		},
 	}
 	table := tablewriter.NewWriter(&buff)
-	table.Header("Total", "Added", "Modified", "Removed")
+	table.Header([]string{"Total", "Added", "Modified", "Removed"})
 	for _, v := range data {
 		_ = table.Append(v)
 	}

pkg/protocols/common/protocolstate/file.go

@@ -2,24 +2,66 @@ package protocolstate
 
 import (
 	"strings"
+	"sync"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
 var (
 	// LfaAllowed means local file access is allowed
 	LfaAllowed bool
+	lfaMutex   sync.Mutex
 )
 
+// IsLfaAllowed returns whether local file access is allowed
+func IsLfaAllowed(options *types.Options) bool {
+	// Use the global when no options are provided
+	if options == nil {
+		lfaMutex.Lock()
+		defer lfaMutex.Unlock()
+		return LfaAllowed
+	}
+	// Otherwise the specific options
+	dialers, ok := dialers.Get(options.ExecutionId)
+	if ok && dialers != nil {
+		dialers.Lock()
+		defer dialers.Unlock()
+
+		return dialers.LocalFileAccessAllowed
+	}
+	return false
+}
+
+func SetLfaAllowed(options *types.Options) {
+	// TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks.
+	lfaMutex.Lock()
+	if options != nil {
+		LfaAllowed = options.AllowLocalFileAccess
+	}
+	lfaMutex.Unlock()
+}
+
+func GetLfaAllowed(options *types.Options) bool {
+	if options != nil {
+		return options.AllowLocalFileAccess
+	}
+	// TODO: Replace this global with per-options function calls. The big lift is handling the javascript fs module callbacks.
+	lfaMutex.Lock()
+	defer lfaMutex.Unlock()
+	return LfaAllowed
+}
+
 // Normalizepath normalizes path and returns absolute path
 // it returns error if path is not allowed
 // this respects the sandbox rules and only loads files from
 // allowed directories
 func NormalizePath(filePath string) (string, error) {
-	// TODO: this should be tied to executionID
-	if LfaAllowed {
+	// TODO: this should be tied to executionID using *types.Options
+	if IsLfaAllowed(nil) {
+		// if local file access is allowed, we can return the absolute path
 		return filePath, nil
 	}
 	cleaned, err := fileutil.ResolveNClean(filePath, config.DefaultConfig.GetTemplateDir())

pkg/protocols/common/protocolstate/headless.go

@@ -74,18 +74,6 @@ func InitHeadless(options *types.Options) {
 	}
 }
 
-// AllowLocalFileAccess returns whether local file access is allowed
-func IsLfaAllowed(options *types.Options) bool {
-	dialers, ok := dialers.Get(options.ExecutionId)
-	if ok && dialers != nil {
-		dialers.Lock()
-		defer dialers.Unlock()
-
-		return dialers.LocalFileAccessAllowed
-	}
-	return false
-}
-
 func IsRestrictLocalNetworkAccess(options *types.Options) bool {
 	dialers, ok := dialers.Get(options.ExecutionId)
 	if ok && dialers != nil {

pkg/protocols/common/protocolstate/state.go

@@ -200,9 +200,7 @@ func initDialers(options *types.Options) error {
 
 	StartActiveMemGuardian(context.Background())
 
-	// TODO: this should be tied to executionID
-	// overidde global settings with latest options
-	LfaAllowed = options.AllowLocalFileAccess
+	SetLfaAllowed(options)
 
 	return nil
 }

pkg/protocols/dns/dnsclientpool/clientpool.go

@@ -11,9 +11,11 @@ import (
 )
 
 var (
-	poolMutex    *sync.RWMutex
+	poolMutex  *sync.RWMutex
+	clientPool map[string]*retryabledns.Client
+
 	normalClient *retryabledns.Client
-	clientPool   map[string]*retryabledns.Client
+	m            sync.Mutex
 )
 
 // defaultResolvers contains the list of resolvers known to be trusted.
@@ -26,6 +28,9 @@ var defaultResolvers = []string{
 
 // Init initializes the client pool implementation
 func Init(options *types.Options) error {
+	m.Lock()
+	defer m.Unlock()
+
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil
@@ -45,6 +50,12 @@ func Init(options *types.Options) error {
 	return nil
 }
 
+func getNormalClient() *retryabledns.Client {
+	m.Lock()
+	defer m.Unlock()
+	return normalClient
+}
+
 // Configuration contains the custom configuration options for a client
 type Configuration struct {
 	// Retries contains the retries for the dns client
@@ -71,7 +82,7 @@ func (c *Configuration) Hash() string {
 // Get creates or gets a client for the protocol based on custom configuration
 func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
 	if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
-		return normalClient, nil
+		return getNormalClient(), nil
 	}
 	hash := configuration.Hash()
 	poolMutex.RLock()

pkg/protocols/http/httpclientpool/clientpool.go

@@ -154,16 +154,16 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client {
 		return dialers.RawHTTPClient
 	}
 
-	rawHttpOptions := rawhttp.DefaultOptions
+	rawHttpOptionsCopy := *rawhttp.DefaultOptions
 	if options.Options.AliveHttpProxy != "" {
-		rawHttpOptions.Proxy = options.Options.AliveHttpProxy
+		rawHttpOptionsCopy.Proxy = options.Options.AliveHttpProxy
 	} else if options.Options.AliveSocksProxy != "" {
-		rawHttpOptions.Proxy = options.Options.AliveSocksProxy
+		rawHttpOptionsCopy.Proxy = options.Options.AliveSocksProxy
 	} else if dialers.Fastdialer != nil {
-		rawHttpOptions.FastDialer = dialers.Fastdialer
+		rawHttpOptionsCopy.FastDialer = dialers.Fastdialer
 	}
-	rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout
-	dialers.RawHTTPClient = rawhttp.NewClient(rawHttpOptions)
+	rawHttpOptionsCopy.Timeout = options.Options.GetTimeouts().HttpTimeout
+	dialers.RawHTTPClient = rawhttp.NewClient(&rawHttpOptionsCopy)
 	return dialers.RawHTTPClient
 }
 

pkg/protocols/protocols.go

@@ -3,7 +3,6 @@ package protocols
 import (
 	"context"
 	"encoding/base64"
-	"sync"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -139,8 +138,6 @@ type ExecutorOptions struct {
 	Logger *gologger.Logger
 	// CustomFastdialer is a fastdialer dialer instance
 	CustomFastdialer *fastdialer.Dialer
-
-	m sync.Mutex
 }
 
 // todo: centralizing components is not feasible with current clogged architecture
@@ -198,6 +195,11 @@ func (e *ExecutorOptions) HasTemplateCtx(input *contextargs.MetaInput) bool {
 // GetTemplateCtx returns template context for given input
 func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context {
 	scanId := input.GetScanHash(e.TemplateID)
+	if e.templateCtxStore == nil {
+		// if template context store is not initialized create it
+		e.CreateTemplateCtxStore()
+	}
+	// get template context from store
 	templateCtx, ok := e.templateCtxStore.Get(scanId)
 	if !ok {
 		// if template context does not exist create new and add it to store and return it
@@ -444,14 +446,49 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	if e == nil || n == nil || n.Options == nil {
 		return
 	}
-	execID := n.Options.GetExecutionID()
-	e.SetExecutionID(execID)
-}
 
-// ApplyNewEngineOptions updates an existing ExecutorOptions with options from a new engine. This
-// handles things like the ExecutionID that need to be updated.
-func (e *ExecutorOptions) SetExecutionID(executorId string) {
-	e.m.Lock()
-	defer e.m.Unlock()
-	e.Options.SetExecutionID(executorId)
+	// The types.Options include the ExecutionID among other things
+	e.Options = n.Options.Copy()
+
+	// Keep the template-specific fields, but replace the rest
+	/*
+		e.TemplateID = n.TemplateID
+		e.TemplatePath = n.TemplatePath
+		e.TemplateInfo = n.TemplateInfo
+		e.TemplateVerifier = n.TemplateVerifier
+		e.RawTemplate = n.RawTemplate
+		e.Variables = n.Variables
+		e.Constants = n.Constants
+	*/
+	e.Output = n.Output
+	e.Options = n.Options
+	e.IssuesClient = n.IssuesClient
+	e.Progress = n.Progress
+	e.RateLimiter = n.RateLimiter
+	e.Catalog = n.Catalog
+	e.ProjectFile = n.ProjectFile
+	e.Browser = n.Browser
+	e.Interactsh = n.Interactsh
+	e.HostErrorsCache = n.HostErrorsCache
+	e.StopAtFirstMatch = n.StopAtFirstMatch
+	e.ExcludeMatchers = n.ExcludeMatchers
+	e.InputHelper = n.InputHelper
+	e.FuzzParamsFrequency = n.FuzzParamsFrequency
+	e.FuzzStatsDB = n.FuzzStatsDB
+	e.DoNotCache = n.DoNotCache
+	e.Colorizer = n.Colorizer
+	e.WorkflowLoader = n.WorkflowLoader
+	e.ResumeCfg = n.ResumeCfg
+	e.ProtocolType = n.ProtocolType
+	e.Flow = n.Flow
+	e.IsMultiProtocol = n.IsMultiProtocol
+	e.templateCtxStore = n.templateCtxStore
+	e.JsCompiler = n.JsCompiler
+	e.AuthProvider = n.AuthProvider
+	e.TemporaryDirectory = n.TemporaryDirectory
+	e.Parser = n.Parser
+	e.ExportReqURLPattern = n.ExportReqURLPattern
+	e.GlobalMatchers = n.GlobalMatchers
+	e.Logger = n.Logger
+	e.CustomFastdialer = n.CustomFastdialer
 }

pkg/protocols/whois/rdapclientpool/clientpool.go

@@ -30,6 +30,12 @@ func Init(options *types.Options) error {
 	return nil
 }
 
+func getNormalClient() *rdap.Client {
+	m.Lock()
+	defer m.Unlock()
+	return normalClient
+}
+
 // Configuration contains the custom configuration options for a client - placeholder
 type Configuration struct{}
 
@@ -40,7 +46,5 @@ func (c *Configuration) Hash() string {
 
 // Get creates or gets a client for the protocol based on custom configuration
 func Get(options *types.Options, configuration *Configuration) (*rdap.Client, error) {
-	m.Lock()
-	defer m.Unlock()
-	return normalClient, nil
+	return getNormalClient(), nil
 }