Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions lib/multi.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
// in ThreadSafeNucleiEngine
type unsafeOptions struct {
executerOpts *protocols.ExecutorOptions
engine *core.Engine
}

// createEphemeralObjects creates ephemeral nuclei objects/instances/types
Expand Down Expand Up @@ -52,8 +51,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
opts.RateLimitDuration = time.Second
}
u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration)
u.engine = core.New(opts)
u.engine.SetExecuterOptions(u.executerOpts)
return u, nil
}

Expand Down Expand Up @@ -145,7 +142,9 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
}
unsafeOpts.executerOpts.WorkflowLoader = workflowLoader

store, err := loader.New(loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts))
loaderConfig := loader.NewConfig(tmpEngine.opts, e.eng.catalog, unsafeOpts.executerOpts)
loaderConfig.MetadataIndex = e.eng.getMetadataIndex()
store, err := loader.New(loaderConfig)
if err != nil {
return errkit.Wrapf(err, "Could not create loader client: %s", err)
}
Expand Down
73 changes: 73 additions & 0 deletions lib/multi_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package nuclei

import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"

"github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
)

func BenchmarkExecuteNucleiWithOptsCtx(b *testing.B) {
templatePath := filepath.Join(b.TempDir(), "benchmark.yaml")
err := os.WriteFile(templatePath, []byte(`id: benchmark-execute-nuclei-with-opts-ctx

info:
name: Benchmark ExecuteNucleiWithOptsCtx
author: pdteam
severity: info

http:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: status
status:
- 418
`), 0o600)
if err != nil {
b.Fatalf("could not write benchmark template: %s", err)
}

server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
b.Cleanup(server.Close)

metadataIndex, err := index.NewIndex(b.TempDir())
if err != nil {
b.Fatalf("could not create benchmark metadata index: %s", err)
}

engine, err := NewThreadSafeNucleiEngineCtx(
b.Context(),
DisableUpdateCheck(),
WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}),
)
if err != nil {
b.Fatalf("could not create thread-safe nuclei engine: %s", err)
}
b.Cleanup(engine.Close)

// Keep benchmark metadata isolated from the user's persistent cache.
engine.eng.metadataIndexOnce.Do(func() {
engine.eng.metadataIndex = metadataIndex
})

targets := []string{server.URL}
if err := engine.ExecuteNucleiWithOptsCtx(b.Context(), targets); err != nil {
b.Fatalf("could not warm up nuclei execution: %s", err)
}

b.ReportAllocs()
b.ResetTimer()

for b.Loop() {
if err := engine.ExecuteNucleiWithOptsCtx(b.Context(), targets); err != nil {
b.Fatalf("could not execute nuclei: %s", err)
}
}
}
33 changes: 32 additions & 1 deletion lib/sdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v3/pkg/core"
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
Expand Down Expand Up @@ -83,6 +84,9 @@ type NucleiEngine struct {
ownsParser bool // true when the engine created the parser and may purge it on Close
authprovider authprovider.AuthProvider

metadataIndex *metadataindex.Index
metadataIndexOnce sync.Once

// unexported meta options
opts *types.Options
interactshOpts *interactsh.Options
Expand All @@ -100,6 +104,24 @@ type NucleiEngine struct {
tmpDir string
}

func (e *NucleiEngine) getMetadataIndex() *metadataindex.Index {
e.metadataIndexOnce.Do(func() {
metadataIndex, err := metadataindex.NewDefaultIndex()
if err != nil {
e.Logger.Warning().Msgf("Could not create metadata cache: %v", err)

return
}

e.metadataIndex = metadataIndex
if err := e.metadataIndex.Load(); err != nil {
e.Logger.Warning().Msgf("Could not load metadata cache: %v", err)
}
})

return e.metadataIndex
}

// LoadAllTemplates loads all nuclei template based on given options
func (e *NucleiEngine) LoadAllTemplates() error {
workflowLoader, err := workflow.NewLoader(e.executerOpts)
Expand All @@ -108,7 +130,11 @@ func (e *NucleiEngine) LoadAllTemplates() error {
}
e.executerOpts.WorkflowLoader = workflowLoader

e.store, err = loader.New(loader.NewConfig(e.opts, e.catalog, e.executerOpts))
loaderConfig := loader.NewConfig(e.opts, e.catalog, e.executerOpts)
if e.mode == threadSafe {
loaderConfig.MetadataIndex = e.getMetadataIndex()
}
e.store, err = loader.New(loaderConfig)
if err != nil {
return errkit.Wrapf(err, "Could not create loader client: %s", err)
}
Expand Down Expand Up @@ -239,6 +265,11 @@ func (e *NucleiEngine) closeInternal() {
if e.httpxClient != nil {
_ = e.httpxClient.Close()
}
if e.metadataIndex != nil {
if err := e.metadataIndex.Save(); err != nil {
e.Logger.Warning().Msgf("Could not save metadata cache: %v", err)
}
}
Comment on lines +268 to +272

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🔴 Critical | ⚡ Quick win

Unsynchronized read of e.metadataIndex races with concurrent getMetadataIndex() writers.

e.metadataIndex is written inside metadataIndexOnce.Do(...) (lines 107-123), but closeInternal reads the raw field directly instead of going through the same sync.Once-guarded getter. If Close() runs concurrently with an in-flight ExecuteNucleiWithOptsCtx call that's still initializing the index (a realistic scenario for a "thread-safe" engine), this is a data race per the Go memory model.

🔒 Proposed fix: read through the synchronized getter
-	if e.metadataIndex != nil {
-		if err := e.metadataIndex.Save(); err != nil {
+	if metadataIndex := e.getMetadataIndex(); metadataIndex != nil {
+		if err := metadataIndex.Save(); err != nil {
 			e.Logger.Warning().Msgf("Could not save metadata cache: %v", err)
 		}
 	}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if e.metadataIndex != nil {
if err := e.metadataIndex.Save(); err != nil {
e.Logger.Warning().Msgf("Could not save metadata cache: %v", err)
}
}
if metadataIndex := e.getMetadataIndex(); metadataIndex != nil {
if err := metadataIndex.Save(); err != nil {
e.Logger.Warning().Msgf("Could not save metadata cache: %v", err)
}
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@lib/sdk.go` around lines 268 - 272, Update closeInternal to obtain the
metadata index through the existing synchronized getMetadataIndex() accessor
before saving it, rather than reading e.metadataIndex directly. Preserve the
current nil check and warning behavior while ensuring the read uses the same
synchronization as metadataIndexOnce.Do.

if e.tmpDir != "" {
_ = os.RemoveAll(e.tmpDir)
}
Expand Down
64 changes: 45 additions & 19 deletions pkg/catalog/index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package index

import (
"encoding/gob"
"errors"
"maps"
"os"
"path/filepath"
Expand Down Expand Up @@ -34,6 +35,7 @@ type Index struct {
cacheFile string
mu sync.RWMutex
version int
dirty bool
}

// cacheSnapshot represents the serialized cache structure.
Expand Down Expand Up @@ -105,20 +107,26 @@ func NewDefaultIndex() (*Index, error) {
// Get retrieves metadata for a template path, validating freshness via mtime.
func (i *Index) Get(path string) (*Metadata, bool) {
i.mu.RLock()
defer i.mu.RUnlock()

metadata, found := i.cache.GetIfPresent(path)
i.mu.RUnlock()

if !found {
return nil, false
}

if !metadata.IsValid() {
go i.Delete(path)
if metadata.IsValid() {
return metadata.clone(), true
}

return nil, false
i.mu.Lock()
current, found := i.cache.GetIfPresent(path)
if found && current == metadata {
i.cache.Invalidate(path)
i.dirty = true
}
i.mu.Unlock()

return metadata, true
return nil, false
}

// Set stores metadata for a template path.
Expand All @@ -132,7 +140,10 @@ func (i *Index) Set(path string, metadata *Metadata) (*Metadata, bool) {
i.mu.Lock()
defer i.mu.Unlock()

return i.cache.Set(path, metadata)
cached, ok := i.cache.Set(path, metadata.clone())
i.dirty = true

return cached.clone(), ok
}

// SetFromTemplate extracts metadata from a parsed template and stores it.
Expand Down Expand Up @@ -171,7 +182,12 @@ func (i *Index) Delete(path string) {
i.mu.Lock()
defer i.mu.Unlock()

if _, found := i.cache.GetIfPresent(path); !found {
return
}

i.cache.Invalidate(path)
i.dirty = true
}

// Size returns the number of cached entries.
Expand All @@ -188,18 +204,22 @@ func (i *Index) Clear() {
defer i.mu.Unlock()

i.cache.InvalidateAll()
i.dirty = true
}

// Save persists the cache to disk using gob encoding.
func (i *Index) Save() error {
i.mu.RLock()
defer i.mu.RUnlock()
i.mu.Lock()
defer i.mu.Unlock()

if !i.dirty {
return nil
}

snapshot := &cacheSnapshot{
Version: i.version,
Data: make(map[string]*Metadata),
Data: make(map[string]*Metadata, i.cache.EstimatedSize()),
}

maps.Insert(snapshot.Data, i.cache.All())

// NOTE(dwisiswant0): write to temp for atomic op.
Expand Down Expand Up @@ -229,11 +249,20 @@ func (i *Index) Save() error {
return err
}

i.dirty = false

return nil
}

// Load loads the cache from disk using gob decoding.
func (i *Index) Load() error {
i.mu.Lock()
defer i.mu.Unlock()

if i.dirty {
return errors.New("cannot load metadata cache into a modified index")
}

file, err := os.Open(i.cacheFile)
if err != nil {
if os.IsNotExist(err) {
Expand Down Expand Up @@ -261,9 +290,6 @@ func (i *Index) Load() error {
return nil
}

i.mu.Lock()
defer i.mu.Unlock()

for key, value := range snapshot.Data {
i.cache.Set(key, value)
}
Expand Down Expand Up @@ -296,11 +322,8 @@ func (i *Index) FilterFunc(fn FilterFunc) []string {
return i.All()
}

i.mu.RLock()
defer i.mu.RUnlock()

var matched []string
for path, metadata := range i.cache.All() {
for path, metadata := range i.GetAll() {
if fn(metadata) {
matched = append(matched, path)
}
Expand All @@ -327,7 +350,10 @@ func (i *Index) GetAll() map[string]*Metadata {
i.mu.RLock()
defer i.mu.RUnlock()

result := maps.Collect(i.cache.All())
result := make(map[string]*Metadata, i.cache.EstimatedSize())
for path, metadata := range i.cache.All() {
result[path] = metadata.clone()
}

return result
}
Expand Down
Loading
Loading