From e8c0a8e25f6ead0d885a57b1829afcac2c40e795 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 11 Jun 2025 19:10:05 +0530 Subject: [PATCH 1/7] bugfix: fix memory blowup using previousEvent for multi-proto execution --- pkg/tmplexec/generic/exec.go | 16 ++++----------- pkg/tmplexec/multiproto/multi.go | 14 ++----------- pkg/tmplexec/utils/utils.go | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 pkg/tmplexec/utils/utils.go diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go index c017810e75..25f88262e3 100644 --- a/pkg/tmplexec/generic/exec.go +++ b/pkg/tmplexec/generic/exec.go @@ -1,13 +1,13 @@ package generic import ( - "strings" "sync/atomic" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/scan" + "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/utils" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -64,17 +64,9 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error { // ideally this should never happen since protocol exits on error and callback is not called return } - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - _ = previous.Set(builder.String(), v) - builder.Reset() - } - } + + utils.FillPreviousEvent(req.GetID(), event, previous) + if event.HasOperatorResult() { g.results.CompareAndSwap(false, true) } diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index d50e168aca..ef029861a3 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -2,7 +2,6 @@ package multiproto import ( "strconv" - "strings" "sync/atomic" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -10,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/utils" mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -90,17 +90,7 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { return } - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - _ = previous.Set(builder.String(), v) - builder.Reset() - } - } + utils.FillPreviousEvent(req.GetID(), event, previous) // log event and generate result for the event ctx.LogEvent(event) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go new file mode 100644 index 0000000000..64bba64202 --- /dev/null +++ b/pkg/tmplexec/utils/utils.go @@ -0,0 +1,34 @@ +package utils + +import ( + "regexp" + "strings" + + "github.com/projectdiscovery/nuclei/v3/pkg/output" + mapsutil "github.com/projectdiscovery/utils/maps" +) + +var ( + reqTypeWithIndexRegex = regexp.MustCompile(`^(?:dns|http|headless|tcp|ssl|websocket|whois|code|javascript)_\d+_`) +) + +// FillPreviousEvent is a helper function to get the previous event from the event +// without leading to duplicate prefixes +func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { + if ID == "" { + return + } + + builder := &strings.Builder{} + for k, v := range event.InternalEvent { + if reqTypeWithIndexRegex.MatchString(k) { + _ = previous.Set(k, v) + continue + } + builder.WriteString(ID) + builder.WriteString("_") + builder.WriteString(k) + _ = previous.Set(builder.String(), v) + builder.Reset() + } +} From be696cc502ad0b81f4ea5d3551cce9330e7a0c5d Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 11 Jun 2025 21:15:35 +0700 Subject: [PATCH 2/7] refactor(tmplexec): uses supported protocol types Signed-off-by: Dwi Siswanto --- pkg/tmplexec/utils/utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go index 64bba64202..9448c956b8 100644 --- a/pkg/tmplexec/utils/utils.go +++ b/pkg/tmplexec/utils/utils.go @@ -5,11 +5,12 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" mapsutil "github.com/projectdiscovery/utils/maps" ) var ( - reqTypeWithIndexRegex = regexp.MustCompile(`^(?:dns|http|headless|tcp|ssl|websocket|whois|code|javascript)_\d+_`) + reqTypeWithIndexRegex = regexp.MustCompile(`^(` + strings.Join(types.SupportedProtocolsStrings(), "|") + `)_\d+_`) ) // FillPreviousEvent is a helper function to get the previous event from the event From c0a6f5ab053e1ec9c63d6885a6b377b7c117c497 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 11 Jun 2025 21:28:07 +0700 Subject: [PATCH 3/7] add co-author Co-authored-by: Nakul Bharti Signed-off-by: Dwi Siswanto From 7b21de390cf7bcccb28227ef34f763d85fab4960 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 12 Jun 2025 09:27:32 +0700 Subject: [PATCH 4/7] refactor(tmplexec): mv builder inside loop scope Signed-off-by: Dwi Siswanto --- pkg/tmplexec/utils/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go index 9448c956b8..b9d9cbbabd 100644 --- a/pkg/tmplexec/utils/utils.go +++ b/pkg/tmplexec/utils/utils.go @@ -20,8 +20,8 @@ func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous * return } - builder := &strings.Builder{} for k, v := range event.InternalEvent { + var builder strings.Builder if reqTypeWithIndexRegex.MatchString(k) { _ = previous.Set(k, v) continue @@ -30,6 +30,5 @@ func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous * builder.WriteString("_") builder.WriteString(k) _ = previous.Set(builder.String(), v) - builder.Reset() } } From 8f5e2bef536233aa6e25246e20fee091a4a3d5ab Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 16 Jun 2025 18:23:24 +0700 Subject: [PATCH 5/7] refactor(tmplexec): skip existing keys in `FillPreviousEvent` The `FillPreviousEvent` func was modified to prevent overwriting/duplicating entries in the previous map. It now checks if a key `k` from `event.InternalEvent` already exists in the previous map. If it does, the key is skipped. This ensures that if `k` was already set (potentially w/o a prefix), it's not re-added with an `ID_` prefix. Additionally, keys in `event.InternalEvent` that already start with the current `ID_` prefix are also skipped to avoid redundant prefixing. This change simplifies the logic by removing the `reqTypeWithIndexRegex` and directly addresses the potential for duplicate / incorrectly prefixed keys when `event.InternalEvent` grows during protocol request execution. Signed-off-by: Dwi Siswanto --- pkg/tmplexec/utils/utils.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go index b9d9cbbabd..894d1aaa58 100644 --- a/pkg/tmplexec/utils/utils.go +++ b/pkg/tmplexec/utils/utils.go @@ -1,18 +1,12 @@ package utils import ( - "regexp" "strings" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" mapsutil "github.com/projectdiscovery/utils/maps" ) -var ( - reqTypeWithIndexRegex = regexp.MustCompile(`^(` + strings.Join(types.SupportedProtocolsStrings(), "|") + `)_\d+_`) -) - // FillPreviousEvent is a helper function to get the previous event from the event // without leading to duplicate prefixes func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { @@ -21,14 +15,20 @@ func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous * } for k, v := range event.InternalEvent { - var builder strings.Builder - if reqTypeWithIndexRegex.MatchString(k) { - _ = previous.Set(k, v) + if _, ok := previous.Get(k); ok { + continue + } + + if strings.HasPrefix(k, ID+"_") { continue } + + var builder strings.Builder + builder.WriteString(ID) builder.WriteString("_") builder.WriteString(k) + _ = previous.Set(builder.String(), v) } } From ee238963d895c8cff1a09f8ef9d74bd76fb2104f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 16 Jun 2025 18:28:01 +0700 Subject: [PATCH 6/7] chore(tmplexec): naming convention, `ID` => `protoID` Signed-off-by: Dwi Siswanto --- pkg/tmplexec/utils/utils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go index 894d1aaa58..0830b60213 100644 --- a/pkg/tmplexec/utils/utils.go +++ b/pkg/tmplexec/utils/utils.go @@ -9,8 +9,8 @@ import ( // FillPreviousEvent is a helper function to get the previous event from the event // without leading to duplicate prefixes -func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { - if ID == "" { +func FillPreviousEvent(protoID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { + if protoID == "" { return } @@ -19,13 +19,13 @@ func FillPreviousEvent(ID string, event *output.InternalWrappedEvent, previous * continue } - if strings.HasPrefix(k, ID+"_") { + if strings.HasPrefix(k, protoID+"_") { continue } var builder strings.Builder - builder.WriteString(ID) + builder.WriteString(protoID) builder.WriteString("_") builder.WriteString(k) From fbd2d1d0093fa83391c972a879089d0bc337387a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 16 Jun 2025 18:41:42 +0700 Subject: [PATCH 7/7] chore(tmplexec): it's request ID lol sorry Signed-off-by: Dwi Siswanto --- pkg/tmplexec/utils/utils.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tmplexec/utils/utils.go b/pkg/tmplexec/utils/utils.go index 0830b60213..41d717769d 100644 --- a/pkg/tmplexec/utils/utils.go +++ b/pkg/tmplexec/utils/utils.go @@ -9,8 +9,8 @@ import ( // FillPreviousEvent is a helper function to get the previous event from the event // without leading to duplicate prefixes -func FillPreviousEvent(protoID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { - if protoID == "" { +func FillPreviousEvent(reqID string, event *output.InternalWrappedEvent, previous *mapsutil.SyncLockMap[string, any]) { + if reqID == "" { return } @@ -19,13 +19,13 @@ func FillPreviousEvent(protoID string, event *output.InternalWrappedEvent, previ continue } - if strings.HasPrefix(k, protoID+"_") { + if strings.HasPrefix(k, reqID+"_") { continue } var builder strings.Builder - builder.WriteString(protoID) + builder.WriteString(reqID) builder.WriteString("_") builder.WriteString(k)