Skip to content

fix(httpclientpool): rebuild malformed Location URL #5902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ehsandeep merged 2 commits into from
Dec 19, 2024
Merged

fix(httpclientpool): rebuild malformed Location URL #5902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b4fe1df
fix(httpclientpool): rebuild malformed Location URL
dwisiswant0 Dec 12, 2024
0c11347
Merge branch 'dev' into dwisiswant0/fix/httpclientpool/rebuild-malfor…
dogancanbakir Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion pkg/protocols/http/httpclientpool/clientpool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package httpclientpool
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"net/http/cookiejar"
Expand All @@ -25,6 +26,7 @@ import (
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"
mapsutil "github.com/projectdiscovery/utils/maps"
urlutil "github.com/projectdiscovery/utils/url"
)

var (
Expand Down Expand Up @@ -377,7 +379,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
}
}

func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error {
func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error {
if maxRedirects == 0 {
if len(via) > defaultMaxRedirects {
return http.ErrUseLastResponse
Expand All @@ -388,5 +390,29 @@ func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) e
if len(via) > maxRedirects {
return http.ErrUseLastResponse
}

// NOTE(dwisiswant0): rebuild request URL. See #5900.
if u := req.URL.String(); !isURLEncoded(u) {
parsed, err := urlutil.Parse(u)
if err != nil {
return fmt.Errorf("%w: %w", ErrRebuildURL, err)
}

req.URL = parsed.URL
}
Comment on lines +394 to +402
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Enhance error handling and URL validation

The URL rebuilding logic could be improved in several ways:

  1. Add context to the error message
  2. Validate the rebuilt URL
  3. Consider potential side effects of direct URL modification

Consider this enhanced implementation:

 // NOTE(dwisiswant0): rebuild request URL. See #5900.
 if u := req.URL.String(); !isURLEncoded(u) {
     parsed, err := urlutil.Parse(u)
     if err != nil {
-        return fmt.Errorf("%w: %w", ErrRebuildURL, err)
+        return fmt.Errorf("failed to rebuild URL %q: %w", u, ErrRebuildURL)
     }
+    
+    // Validate the rebuilt URL
+    if parsed.URL.Host == "" || parsed.URL.Scheme == "" {
+        return fmt.Errorf("invalid URL after rebuild %q: missing host or scheme", parsed.URL)
+    }
 
     req.URL = parsed.URL
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// NOTE(dwisiswant0): rebuild request URL. See #5900.
if u := req.URL.String(); !isURLEncoded(u) {
parsed, err := urlutil.Parse(u)
if err != nil {
return fmt.Errorf("%w: %w", ErrRebuildURL, err)
}
req.URL = parsed.URL
}
// NOTE(dwisiswant0): rebuild request URL. See #5900.
if u := req.URL.String(); !isURLEncoded(u) {
parsed, err := urlutil.Parse(u)
if err != nil {
return fmt.Errorf("failed to rebuild URL %q: %w", u, ErrRebuildURL)
}
// Validate the rebuilt URL
if parsed.URL.Host == "" || parsed.URL.Scheme == "" {
return fmt.Errorf("invalid URL after rebuild %q: missing host or scheme", parsed.URL)
}
req.URL = parsed.URL
}


return nil
}

// isURLEncoded is an helper function to check if the URL is already encoded
//
// NOTE(dwisiswant0): shall we move this under `projectdiscovery/utils/urlutil`?
Copy link
Member

@dogancanbakir dogancanbakir Dec 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

++

func isURLEncoded(s string) bool {
decoded, err := url.QueryUnescape(s)
if err != nil {
// If decoding fails, it may indicate a malformed URL/invalid encoding.
return false
}

return decoded != s
}
7 changes: 7 additions & 0 deletions pkg/protocols/http/httpclientpool/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package httpclientpool

import "errors"

var (
ErrRebuildURL = errors.New("could not rebuild request URL")
)