Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix dynamic extractor + payloads edgecase by sending req sequentially #5016

Merged
merged 2 commits into from
Apr 8, 2024
Merged

fix dynamic extractor + payloads edgecase by sending req sequentially #5016

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8358b8d
explicitly handle edgecase #4993 instead of hot fix
tarunKoyalwar Apr 8, 2024
417f555
fix typo
tarunKoyalwar Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 26 additions & 16 deletions pkg/protocols/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
json "github.com/json-iterator/go"
"github.com/pkg/errors"

"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
Expand All @@ -20,7 +21,6 @@ import (
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/env"
fileutil "github.com/projectdiscovery/utils/file"
)

Expand Down Expand Up @@ -436,14 +436,32 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
}
if len(request.Payloads) > 0 {
// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
if protocolstate.IsLowOnMemory() {
request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
// Due to a known issue (https://github.com/projectdiscovery/nuclei/issues/5015),
// dynamic extractors cannot be used with payloads. To address this,
// execution is handled by the standard engine without concurrency,
// achieved by setting the thread count to 0.

// this limitation will be removed once we have a better way to handle dynamic extractors with payloads
hasMultipleRequests := false
if len(request.Raw)+len(request.Path) > 1 {
hasMultipleRequests = true
}
// if we have payloads, adjust threads if none specified
// We only do this in case we have more payload requests than the
// specified concurrency threshold.
if request.generator.NewIterator().Total() > PayloadAutoConcurrencyThreshold {
// look for dynamic extractor ( internal: true with named extractor)
hasNamedInternalExtractor := false
for _, extractor := range request.Extractors {
if extractor.Internal && extractor.Name != "" {
hasNamedInternalExtractor = true
break
}
}
if hasNamedInternalExtractor && hasMultipleRequests {
gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
request.Threads = 0
} else {
// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
if protocolstate.IsLowOnMemory() {
request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
}
request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
}
}
Expand Down Expand Up @@ -472,11 +490,3 @@ func (request *Request) Requests() int {
}
return len(request.Path)
}

// PayloadAutoConcurrencyThreshold is the threshold for auto adjusting concurrency
// for payloads in a template.
var PayloadAutoConcurrencyThreshold int

func init() {
PayloadAutoConcurrencyThreshold = env.GetEnvOrDefault[int]("NUCLEI_PAYLOAD_AUTO_CONCURRENCY_THRESHOLD", 100)
}
Loading