Merge pull request #5331 from projectdiscovery/use_containsall

use `stringsutil.ContainsAll`

cmd/integration-test/loader.go

@@ -172,7 +172,14 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404")
+	configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+	err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
+	if err != nil {
+		return err
+	}
+	defer os.Remove("test-config.yaml")
+
+	_, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -188,7 +195,14 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
+	configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
+	err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
+	if err != nil {
+		return err
+	}
+	defer os.Remove("test-config.yaml")
+
+	_, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}

go.mod

@@ -83,7 +83,7 @@ require (
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
 	github.com/projectdiscovery/goflags v0.1.56
-	github.com/projectdiscovery/gologger v1.1.12
+	github.com/projectdiscovery/gologger v1.1.13
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.3
@@ -95,7 +95,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.8
 	github.com/projectdiscovery/useragent v0.0.57
-	github.com/projectdiscovery/utils v0.1.5
+	github.com/projectdiscovery/utils v0.1.6
 	github.com/projectdiscovery/wappalyzergo v0.1.6
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1

go.sum

@@ -846,8 +846,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
 github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
 github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
-github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
-github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
+github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E=
+github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
@@ -888,8 +888,8 @@ github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiM
 github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
 github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4=
 github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc=
-github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0=
-github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4=
+github.com/projectdiscovery/utils v0.1.6 h1:fzdN+oCTics3hPPOEV1FWqf/Hpenav5Ml/73aEwpTZc=
+github.com/projectdiscovery/utils v0.1.6/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc=
 github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
 github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

integration_tests/protocols/code/pre-condition.yaml

@@ -23,4 +23,4 @@ code:
       - type: dsl
         dsl:
           - true
-# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a0047304502200307590191cb7c766b6c21e5777d345bdddf7adf9d6da8f7d336d585d9ac4a8b022100fd30fb0c7722778eb3d861d60e721d805925b8d8df2b979ef2104c35ec57d5cb:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a0047304502203fe1d7d52bc2a41886d576a90c82c3be42078baaa4b46e1f3d8519665d6f88b202210081feb82c41150c5b218e226fc4f299ded19f42ba01ef34ba60b0634b4ea6ee12:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100afb5ebff14a40e7f9b679ffc4d93ce7849e33eb398ebb47f2e757cd24831f9dd02210089ffa21b2763e99ebce95dfc5b91e1e62da4ccdc9d2ad5c48584fa350ba335af:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100939f83e74d43932a5bd792b1fd2c100eec2df60f2b2a8dd56b5c8ef5faa92b17022100f93031b0de373af7d78e623968ea5a2d67c4561ef70e3e6da15aef7e5c853115:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100b8e676ce0c57b60c233a0203539dec20457bbb5f1790d351a5d45405b6668b2602204b1f2fa18e7db099f05329009597ceb2d9b7337562c1a676e8d50ea2f1c6fcbe:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-template-1.yaml

@@ -19,4 +19,4 @@ code:
         regex:
           - 'hello from (.*)'
         group: 1
-# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a00463044022050da011362cf08c2cb81e812c7f86d7282afe0562d4bf00d390f1300d19bc910022029e9d305da69e941ac18797645aecb217abde6557f891e141301b48e89a3c0cd:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/workflow/code-template-2.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from first"
-# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4b0a00483046022100b3b8759c0df028455eb59b1433ac240e5d4604b011bb0c63680bd3cc159ac6f0022100f44aa11b640d11ad0e2902897f4eb51666ab3cd83c31dfd2590f6e43391e39b0:4a3eb6b4988d95847d4203be25ed1d46

pkg/input/provider/http/multiformat.go

@@ -66,10 +66,10 @@ func (i *HttpInputProvider) Count() int64 {
 // Iterate over all inputs in order
 func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) {
 	err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool {
-		return callback(&contextargs.MetaInput{
-			ReqResp: request,
-			Input:   request.URL.String(),
-		})
+		metaInput := contextargs.NewMetaInput()
+		metaInput.ReqResp = request
+		metaInput.Input = request.URL.String()
+		return callback(metaInput)
 	})
 	if err != nil {
 		gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err)

pkg/input/provider/interface.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	configTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var (
@@ -32,7 +33,7 @@ func IsErrNotImplemented(err error) bool {
 	if err == nil {
 		return false
 	}
-	if strings.Contains(err.Error(), "provider") && strings.Contains(err.Error(), "does not implement") {
+	if stringsutil.ContainsAll(err.Error(), "provider", "does not implement") {
 		return true
 	}
 	return false

pkg/input/provider/list/hmap.go

@@ -122,7 +122,7 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput)
 		})
 	}
 	callbackFunc := func(k, _ []byte) error {
-		metaInput := &contextargs.MetaInput{}
+		metaInput := contextargs.NewMetaInput()
 		if err := metaInput.Unmarshal(string(k)); err != nil {
 			return err
 		}
@@ -153,14 +153,16 @@ func (i *ListInputProvider) Set(value string) {
 			}
 			return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL)
 		})
-		metaInput := &contextargs.MetaInput{Input: URL}
+		metaInput := contextargs.NewMetaInput()
+		metaInput.Input = URL
 		i.setItem(metaInput)
 		return
 	}
 
 	// Check if input is ip or hostname
 	if iputil.IsIP(urlx.Hostname()) {
-		metaInput := &contextargs.MetaInput{Input: URL}
+		metaInput := contextargs.NewMetaInput()
+		metaInput.Input = URL
 		i.setItem(metaInput)
 		return
 	}
@@ -181,7 +183,9 @@ func (i *ListInputProvider) Set(value string) {
 					if ip == "" {
 						continue
 					}
-					metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+					metaInput := contextargs.NewMetaInput()
+					metaInput.Input = value
+					metaInput.CustomIP = ip
 					i.setItem(metaInput)
 				}
 				return
@@ -211,11 +215,13 @@ func (i *ListInputProvider) Set(value string) {
 	}
 
 	for _, ip := range ips {
+		metaInput := contextargs.NewMetaInput()
 		if ip != "" {
-			metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip}
+			metaInput.Input = URL
+			metaInput.CustomIP = ip
 			i.setItem(metaInput)
 		} else {
-			metaInput := &contextargs.MetaInput{Input: URL}
+			metaInput.Input = URL
 			i.setItem(metaInput)
 		}
 	}
@@ -353,7 +359,8 @@ func (i *ListInputProvider) scanInputFromReader(reader io.Reader) {
 
 // isExcluded checks if a URL is in the exclusion list
 func (i *ListInputProvider) isExcluded(URL string) bool {
-	metaInput := &contextargs.MetaInput{Input: URL}
+	metaInput := contextargs.NewMetaInput()
+	metaInput.Input = URL
 	key, err := metaInput.MarshalString()
 	if err != nil {
 		gologger.Warning().Msgf("%s\n", err)
@@ -378,14 +385,16 @@ func (i *ListInputProvider) Del(value string) {
 			}
 			return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL)
 		})
-		metaInput := &contextargs.MetaInput{Input: URL}
+		metaInput := contextargs.NewMetaInput()
+		metaInput.Input = URL
 		i.delItem(metaInput)
 		return
 	}
 
 	// Check if input is ip or hostname
 	if iputil.IsIP(urlx.Hostname()) {
-		metaInput := &contextargs.MetaInput{Input: URL}
+		metaInput := contextargs.NewMetaInput()
+		metaInput.Input = URL
 		i.delItem(metaInput)
 		return
 	}
@@ -406,7 +415,9 @@ func (i *ListInputProvider) Del(value string) {
 					if ip == "" {
 						continue
 					}
-					metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+					metaInput := contextargs.NewMetaInput()
+					metaInput.Input = value
+					metaInput.CustomIP = ip
 					i.delItem(metaInput)
 				}
 				return
@@ -436,11 +447,13 @@ func (i *ListInputProvider) Del(value string) {
 	}
 
 	for _, ip := range ips {
+		metaInput := contextargs.NewMetaInput()
 		if ip != "" {
-			metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip}
+			metaInput.Input = URL
+			metaInput.CustomIP = ip
 			i.delItem(metaInput)
 		} else {
-			metaInput := &contextargs.MetaInput{Input: URL}
+			metaInput.Input = URL
 			i.delItem(metaInput)
 		}
 	}
@@ -514,7 +527,8 @@ func (i *ListInputProvider) addTargets(targets []string) {
 
 func (i *ListInputProvider) removeTargets(targets []string) {
 	for _, target := range targets {
-		metaInput := &contextargs.MetaInput{Input: target}
+		metaInput := contextargs.NewMetaInput()
+		metaInput.Input = target
 		i.delItem(metaInput)
 	}
 }

pkg/input/provider/list/hmap_test.go

@@ -40,7 +40,7 @@ func Test_expandCIDR(t *testing.T) {
 		// scan
 		got := []string{}
 		input.hostMap.Scan(func(k, _ []byte) error {
-			var metainput contextargs.MetaInput
+			metainput := contextargs.NewMetaInput()
 			if err := metainput.Unmarshal(string(k)); err != nil {
 				return err
 			}
@@ -141,7 +141,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 		// scan
 		got := []string{}
 		input.hostMap.Scan(func(k, v []byte) error {
-			var metainput contextargs.MetaInput
+			metainput := contextargs.NewMetaInput()
 			if err := metainput.Unmarshal(string(k)); err != nil {
 				return err
 			}
@@ -184,7 +184,7 @@ func Test_expandASNInputValue(t *testing.T) {
 		// scan the hmap
 		got := []string{}
 		input.hostMap.Scan(func(k, v []byte) error {
-			var metainput contextargs.MetaInput
+			metainput := contextargs.NewMetaInput()
 			if err := metainput.Unmarshal(string(k)); err != nil {
 				return err
 			}

pkg/input/provider/simple.go

@@ -43,7 +43,9 @@ func (s *SimpleInputProvider) Iterate(callback func(value *contextargs.MetaInput
 
 // Set adds an item to the input provider
 func (s *SimpleInputProvider) Set(value string) {
-	s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
+	metaInput := contextargs.NewMetaInput()
+	metaInput.Input = value
+	s.Inputs = append(s.Inputs, metaInput)
 }
 
 // SetWithProbe adds an item to the input provider with HTTP probing
@@ -52,13 +54,17 @@ func (s *SimpleInputProvider) SetWithProbe(value string, probe types.InputLivene
 	if err != nil {
 		return err
 	}
-	s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: probedValue})
+	metaInput := contextargs.NewMetaInput()
+	metaInput.Input = probedValue
+	s.Inputs = append(s.Inputs, metaInput)
 	return nil
 }
 
 // SetWithExclusions adds an item to the input provider if it doesn't match any of the exclusions
 func (s *SimpleInputProvider) SetWithExclusions(value string) error {
-	s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
+	metaInput := contextargs.NewMetaInput()
+	metaInput.Input = value
+	s.Inputs = append(s.Inputs, metaInput)
 	return nil
 }
 

pkg/protocols/common/contextargs/contextargs.go

@@ -50,9 +50,11 @@ func NewWithInput(ctx context.Context, input string) *Context {
 	if err != nil {
 		gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err)
 	}
+	metaInput := NewMetaInput()
+	metaInput.Input = input
 	return &Context{
 		ctx:       ctx,
-		MetaInput: &MetaInput{Input: input},
+		MetaInput: metaInput,
 		CookieJar: jar,
 		args: &mapsutil.SyncLockMap[string, interface{}]{
 			Map:      make(map[string]interface{}),

pkg/protocols/common/contextargs/metainput.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"sync"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
@@ -24,6 +25,12 @@ type MetaInput struct {
 
 	// ReqResp is the raw request for the input
 	ReqResp *types.RequestResponse `json:"raw-request,omitempty"`
+
+	mu *sync.Mutex
+}
+
+func NewMetaInput() *MetaInput {
+	return &MetaInput{mu: &sync.Mutex{}}
 }
 
 func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) {
@@ -135,10 +142,9 @@ func (metaInput *MetaInput) Unmarshal(data string) error {
 }
 
 func (metaInput *MetaInput) Clone() *MetaInput {
-	input := &MetaInput{
-		Input:    metaInput.Input,
-		CustomIP: metaInput.CustomIP,
-	}
+	input := NewMetaInput()
+	input.Input = metaInput.Input
+	input.CustomIP = metaInput.CustomIP
 	if metaInput.ReqResp != nil {
 		input.ReqResp = metaInput.ReqResp.Clone()
 	}
@@ -160,6 +166,9 @@ func (metaInput *MetaInput) GetScanHash(templateId string) string {
 	// there may be some cases where metainput is changed ex: while executing self-contained template etc
 	// but that totally changes the scanID/hash so to avoid that we compute hash only once
 	// and reuse it for all subsequent calls
+	metaInput.mu.Lock()
+	defer metaInput.mu.Unlock()
+
 	if metaInput.hash == "" {
 		var rawRequest string
 		if metaInput.ReqResp != nil {