diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go index 9c2cddf688..47b3a9ac13 100644 --- a/pkg/input/types/http.go +++ b/pkg/input/types/http.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/useragent" "github.com/projectdiscovery/utils/conversion" mapsutil "github.com/projectdiscovery/utils/maps" + stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ) @@ -239,25 +240,25 @@ func ParseRawRequest(raw string) (rr *RequestResponse, err error) { method := parts[0] rr.Request.Method = method - // parse relative url - urlx, err := urlutil.ParseRawRelativePath(parts[1], true) + // the request target is normally an origin-form path, but proxy captures and + // .http files use the absolute form, which already carries the authority + var urlx *urlutil.URL + target := parts[1] + if stringsutil.HasPrefixAnyI(target, urlutil.HTTP+urlutil.SchemeSeparator, urlutil.HTTPS+urlutil.SchemeSeparator) { + // urlutil.ParseAbsoluteURL only accepts lowercase schemes; preserve the + // remainder of the request target unchanged. + if scheme, rest, ok := strings.Cut(target, urlutil.SchemeSeparator); ok { + target = strings.ToLower(scheme) + urlutil.SchemeSeparator + rest + } + urlx, err = urlutil.ParseAbsoluteURL(target, true) + } else { + urlx, err = urlutil.ParseRawRelativePath(target, true) + } if err != nil { return nil, fmt.Errorf("failed to parse url: %s", err) } rr.URL = *urlx - // parse host line - hostLine, err := protoReader.ReadLine() - if err != nil { - return nil, fmt.Errorf("failed to read host line: %s", err) - } - sep := strings.Index(hostLine, ":") - if sep <= 0 || sep >= len(hostLine)-1 { - return nil, fmt.Errorf("invalid host line: %s", hostLine) - } - hostLine = hostLine[sep+2:] - rr.URL.Host = hostLine - // parse headers rr.Request.Headers = mapsutil.NewOrderedMap[string, string]() for { @@ -269,11 +270,23 @@ func ParseRawRequest(raw string) (rr *RequestResponse, err error) { // end of headers next is body break } - parts := strings.SplitN(line, ":", 2) - if len(parts) != 2 { + key, value, found := strings.Cut(line, ":") + if !found || key == "" { return nil, fmt.Errorf("invalid header line: %s", line) } - rr.Request.Headers.Set(parts[0], parts[1][1:]) + value = strings.TrimSpace(value) + // Host carries the authority rather than request metadata, and callers + // read it off the URL: retryablehttp derives the wire Host from there, + // and keeping it in the header map would expose it to header fuzzing as + // if it were an ordinary header. + if strings.EqualFold(key, "Host") { + // an absolute request target takes precedence over the Host header + if rr.URL.Host == "" { + rr.URL.Host = value + } + continue + } + rr.Request.Headers.Set(key, value) } // parse body diff --git a/pkg/input/types/http_test.go b/pkg/input/types/http_test.go index 8cc44e39cb..cb407080d9 100644 --- a/pkg/input/types/http_test.go +++ b/pkg/input/types/http_test.go @@ -121,6 +121,139 @@ func TestParseRawRequestBodyTrailingNewlines(t *testing.T) { } } +// Headers used to be parsed positionally: the second line was always taken as +// the Host line and dropped from the header map, and values were read by +// skipping a single byte after the colon. Any request-shaped input (burp, jsonl, +// yaml, openapi) hitting one of these shapes either lost a header silently, +// truncated a value, or crashed the whole scan. +func TestParseRawRequestHeaderParsing(t *testing.T) { + tests := []struct { + name string + raw string + host string + headers map[string]string + }{ + { + name: "host first", + raw: "POST /login HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/json\r\n\r\n{}", + host: "example.com", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "host after other headers", + raw: "POST /login HTTP/1.1\r\nContent-Type: application/json\r\nHost: example.com\r\n\r\n{}", + host: "example.com", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "host absent", + raw: "POST /login HTTP/1.1\r\nContent-Type: application/json\r\n\r\n{}", + host: "", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "no space after colon", + raw: "GET / HTTP/1.1\r\nHost:example.com\r\nX-Token:abc\r\n\r\n", + host: "example.com", + headers: map[string]string{"X-Token": "abc"}, + }, + { + name: "valueless header", + raw: "GET / HTTP/1.1\r\nHost: example.com\r\nX-Empty:\r\n\r\n", + host: "example.com", + headers: map[string]string{"X-Empty": ""}, + }, + { + name: "lowercase host key", + raw: "GET / HTTP/1.1\r\ncontent-type: text/plain\r\nhost: example.com\r\n\r\n", + host: "example.com", + headers: map[string]string{"content-type": "text/plain"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rr *RequestResponse + var err error + require.NotPanics(t, func() { + rr, err = ParseRawRequest(tt.raw) + }) + require.NoError(t, err) + require.Equal(t, tt.host, rr.URL.Host) + + got := map[string]string{} + rr.Request.Headers.Iterate(func(k, v string) bool { + got[k] = v + return true + }) + require.Equal(t, tt.headers, got) + }) + } +} + +// Requests captured through a proxy carry an absolute request target, which the +// parser used to append to the authority as if it were a path. +func TestParseRawRequestAbsoluteTarget(t *testing.T) { + tests := []struct { + name string + raw string + url string + host string + path string + }{ + { + name: "absolute target", + raw: "GET http://example.com/p?q=1 HTTP/1.1\r\nHost: example.com\r\n\r\n", + url: "http://example.com/p?q=1", + }, + { + name: "absolute target wins over host header", + raw: "GET https://example.com/p HTTP/1.1\r\nHost: proxy.internal\r\n\r\n", + url: "https://example.com/p", + }, + { + name: "uppercase scheme absolute target", + raw: "GET HTTP://example.com/p?q=1 HTTP/1.1\r\nHost: example.com\r\n\r\n", + url: "http://example.com/p?q=1", + host: "example.com", + path: "/p", + }, + { + name: "mixed-case https absolute target wins over host", + raw: "GET Https://Example.COM/p HTTP/1.1\r\nHost: proxy.internal\r\n\r\n", + url: "https://Example.COM/p", + host: "Example.COM", + path: "/p", + }, + { + name: "origin form keeps host header", + raw: "GET /p HTTP/1.1\r\nHost: example.com\r\n\r\n", + url: "example.com/p", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rr, err := ParseRawRequest(tt.raw) + require.NoError(t, err) + require.Equal(t, tt.url, rr.URL.String()) + if tt.host != "" { + require.Equal(t, tt.host, rr.URL.Host) + } + if tt.path != "" { + require.Equal(t, tt.path, rr.URL.Path) + } + }) + } +} + +// A header line without a colon is not a header, and must not be mistaken for +// the start of the body. +func TestParseRawRequestRejectsMalformedHeader(t *testing.T) { + _, err := ParseRawRequest("GET / HTTP/1.1\r\nHost: example.com\r\nnot-a-header\r\n\r\n") + require.Error(t, err) +} + func TestUnmarshalJSON(t *testing.T) { tests := []struct { name string