From 67571b3cfe654630baf953f6d01daea39417c6a4 Mon Sep 17 00:00:00 2001 From: Matej Smycka Date: Mon, 6 Oct 2025 12:38:29 +0200 Subject: [PATCH 1/2] feat: http(s) probing optimization --- pkg/utils/http_probe.go | 40 ++++++++++++++++++++++++++++-------- pkg/utils/http_probe_test.go | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 pkg/utils/http_probe_test.go diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go index 9e3046fef9..786690c818 100644 --- a/pkg/utils/http_probe.go +++ b/pkg/utils/http_probe.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "net" "net/http" "github.com/projectdiscovery/httpx/common/httpx" @@ -9,15 +10,38 @@ import ( "github.com/projectdiscovery/useragent" ) -var ( - HttpSchemes = []string{"https", "http"} -) +var commonHttpPorts = []string{ + ":80", + ":8080", +} +var httpSchemesHttpFirst = []string{"http", "https"} +var httpSchemesHttpsFirst = []string{"https", "http"} + +// If url contains a port that is commonly used for HTTP, +// return http first, otherwise return https first. +func determineSchemeOrder(input string) []string { + // Full urls (with http(s):// prefix) are not probed + _, port, err := net.SplitHostPort(input) + if err == nil { + // Check if port is a known HTTP port + portWithColon := ":" + port + for _, httpPort := range commonHttpPorts { + if portWithColon == httpPort { + return httpSchemesHttpFirst + } + } + } + return httpSchemesHttpsFirst +} -// ProbeURL probes the scheme for a URL. first HTTPS is tried -// and if any errors occur http is tried. If none succeeds, probing -// is abandoned for such URLs. +// ProbeURL probes the scheme for a URL. +// First http scheme tried is selected based on heuristics +// If none succeeds, probing is abandoned for such URLs. func ProbeURL(input string, httpxclient *httpx.HTTPX) string { - for _, scheme := range HttpSchemes { + + httpSchemesOrdered := determineSchemeOrder(input) + + for _, scheme := range httpSchemesOrdered { formedURL := fmt.Sprintf("%s://%s", scheme, input) req, err := httpxclient.NewRequest(http.MethodHead, formedURL) if err != nil { @@ -39,7 +63,7 @@ type inputLivenessChecker struct { client *httpx.HTTPX } -// ProbeURL probes the scheme for a URL. first HTTPS is tried +// ProbeURL probes the scheme for a URL. func (i *inputLivenessChecker) ProbeURL(input string) (string, error) { return ProbeURL(input, i.client), nil } diff --git a/pkg/utils/http_probe_test.go b/pkg/utils/http_probe_test.go new file mode 100644 index 0000000000..f000a8a811 --- /dev/null +++ b/pkg/utils/http_probe_test.go @@ -0,0 +1,37 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDetermineSchemeOrder(t *testing.T) { + type testCase struct { + input string + expected []string + } + + tests := []testCase{ + // No port or uncommon ports should return https first + {"example.com", []string{"https", "http"}}, + {"example.com:443", []string{"https", "http"}}, + {"127.0.0.1", []string{"https", "http"}}, + {"[fe80::1]:443", []string{"https", "http"}}, + // Common HTTP ports should return http first + {"example.com:80", []string{"http", "https"}}, + {"example.com:8080", []string{"http", "https"}}, + {"127.0.0.1:80", []string{"http", "https"}}, + {"127.0.0.1:8080", []string{"http", "https"}}, + {"fe80::1", []string{"https", "http"}}, + {"[fe80::1]:80", []string{"http", "https"}}, + {"[fe80::1]:8080", []string{"http", "https"}}, + } + + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + actual := determineSchemeOrder(tc.input) + require.Equal(t, tc.expected, actual) + }) + } +} From e0d3bb45ed864a719f7f7f3960bf504a1971011d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 6 Oct 2025 22:38:43 +0200 Subject: [PATCH 2/2] small changes --- pkg/utils/http_probe.go | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go index 786690c818..3cc7d5d04f 100644 --- a/pkg/utils/http_probe.go +++ b/pkg/utils/http_probe.go @@ -8,40 +8,40 @@ import ( "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" "github.com/projectdiscovery/useragent" + sliceutil "github.com/projectdiscovery/utils/slice" ) var commonHttpPorts = []string{ - ":80", - ":8080", + "80", + "8080", +} +var defaultHttpSchemes = []string{ + "https", + "http", +} +var httpFirstSchemes = []string{ + "http", + "https", } -var httpSchemesHttpFirst = []string{"http", "https"} -var httpSchemesHttpsFirst = []string{"https", "http"} -// If url contains a port that is commonly used for HTTP, -// return http first, otherwise return https first. +// determineSchemeOrder for the input func determineSchemeOrder(input string) []string { - // Full urls (with http(s):// prefix) are not probed - _, port, err := net.SplitHostPort(input) - if err == nil { - // Check if port is a known HTTP port - portWithColon := ":" + port - for _, httpPort := range commonHttpPorts { - if portWithColon == httpPort { - return httpSchemesHttpFirst - } + // if input has port that is commonly used for HTTP, return http then https + if _, port, err := net.SplitHostPort(input); err == nil { + if sliceutil.Contains(commonHttpPorts, port) { + return httpFirstSchemes } } - return httpSchemesHttpsFirst + + return defaultHttpSchemes } // ProbeURL probes the scheme for a URL. -// First http scheme tried is selected based on heuristics +// http schemes are selected with heuristics // If none succeeds, probing is abandoned for such URLs. func ProbeURL(input string, httpxclient *httpx.HTTPX) string { - - httpSchemesOrdered := determineSchemeOrder(input) - - for _, scheme := range httpSchemesOrdered { + schemes := determineSchemeOrder(input) + for _, scheme := range schemes { formedURL := fmt.Sprintf("%s://%s", scheme, input) req, err := httpxclient.NewRequest(http.MethodHead, formedURL) if err != nil {