diff --git a/cmd/nuclei/main_test.go b/cmd/nuclei/main_benchmark_test.go similarity index 63% rename from cmd/nuclei/main_test.go rename to cmd/nuclei/main_benchmark_test.go index 01c75d5c80..baf9dbfab5 100644 --- a/cmd/nuclei/main_test.go +++ b/cmd/nuclei/main_benchmark_test.go @@ -3,28 +3,56 @@ package main_test import ( "net/http" "net/http/httptest" + "os" "testing" "time" - "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/internal/runner" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) -func BenchmarkRunEnumeration(b *testing.B) { +var ( + projectPath string + targetURL string +) + +func TestMain(m *testing.M) { + // Set up + + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + os.Setenv("DISABLE_STDOUT", "true") + + var err error + + projectPath, err = os.MkdirTemp("", "nuclei-benchmark-") + if err != nil { + panic(err) + } + dummyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })) - defer dummyServer.Close() - - options := &types.Options{ - RemoteTemplateDomainList: goflags.StringSlice{ - "cloud.projectdiscovery.io", - }, - ProjectPath: "/tmp", - Targets: goflags.StringSlice{dummyServer.URL}, + targetURL = dummyServer.URL + + // Execute tests + + exitCode := m.Run() + + // Tear down + + dummyServer.Close() + _ = os.RemoveAll(projectPath) + os.Unsetenv("DISABLE_STDOUT") + + os.Exit(exitCode) +} + +func getDefaultOptions() *types.Options { + return &types.Options{ + RemoteTemplateDomainList: []string{"cloud.projectdiscovery.io"}, + ProjectPath: projectPath, StatsInterval: 5, MetricsPort: 9092, MaxHostError: 30, @@ -66,22 +94,42 @@ func BenchmarkRunEnumeration(b *testing.B) { // DialerKeepAlive: time.Duration(0), // DASTServerAddress: "localhost:9055", } +} +func runEnumBenchmark(b *testing.B, options *types.Options) { runner.ParseOptions(options) - // Disable logging to reduce benchmark noise. - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - nucleiRunner, err := runner.New(options) if err != nil { b.Fatalf("failed to create runner: %s", err) } + defer nucleiRunner.Close() b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { if err := nucleiRunner.RunEnumeration(); err != nil { - b.Fatalf("RunEnumeration failed: %s", err) + b.Fatalf("%s failed: %s", b.Name(), err) } } } + +func BenchmarkRunEnumeration(b *testing.B) { + // Default case: run enumeration with default options == all nuclei-templates + // b.Run("Default", func(b *testing.B) { + // options := getDefaultOptions() + // options.Targets = []string{targetURL} + + // runEnumBenchmark(b, options) + // }) + + // Case: https://github.com/projectdiscovery/nuclei/pull/6258 + b.Run("Multiproto", func(b *testing.B) { + options := getDefaultOptions() + options.Targets = []string{targetURL} + options.Templates = []string{"./cmd/nuclei/testdata/benchmark/multiproto/"} + + runEnumBenchmark(b, options) + }) +} diff --git a/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-mixed.yaml b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-mixed.yaml new file mode 100644 index 0000000000..d11db6e2d2 --- /dev/null +++ b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-mixed.yaml @@ -0,0 +1,239 @@ +id: basic-template-multiproto-mixed + +info: + name: Test Template Multiple Protocols (Mixed) + author: pdteam + severity: info + +http: + - method: GET + id: first_iter_http + path: + - '{{BaseURL}}/1' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/2' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/3' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/4' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/5' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/6' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/7' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/8' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/9' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /10 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /11 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /12 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /13 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /14 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET / HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /15 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /16 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /17 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /18 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" \ No newline at end of file diff --git a/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-raw.yaml b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-raw.yaml new file mode 100644 index 0000000000..f868fdc575 --- /dev/null +++ b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto-raw.yaml @@ -0,0 +1,292 @@ +id: basic-template-multiproto-raw + +info: + name: Test Template Multiple Protocols RAW + author: pdteam + severity: info + +http: + - raw: + - | + GET /1 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /2 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /3 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /4 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /5 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /6 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /7 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /8 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /9 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /10 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /11 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /12 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /13 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /14 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET / HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /15 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /16 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /17 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" + + - raw: + - | + GET /18 HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + Connection: close + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 + Accept-Language: en-US,en;q=0.9 + + matchers: + - type: word + words: + - "Test is test matcher text" \ No newline at end of file diff --git a/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto.yaml b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto.yaml new file mode 100644 index 0000000000..6a7d37c86c --- /dev/null +++ b/cmd/nuclei/testdata/benchmark/multiproto/basic-template-multiproto.yaml @@ -0,0 +1,170 @@ +id: basic-template-multiproto + +info: + name: Test Template Multiple Protocols + author: pdteam + severity: info + +http: + - method: GET + id: first_iter_http + path: + - '{{BaseURL}}/1' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/2' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/3' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/4' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/5' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/6' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/7' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/8' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/9' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/10' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/11' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/12' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/13' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/14' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/15' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/16' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/17' + + matchers: + - type: word + words: + - "Test is test matcher text" + + - method: GET + path: + - '{{BaseURL}}/18' + + matchers: + - type: word + words: + - "Test is test matcher text" \ No newline at end of file