diff --git a/README.md b/README.md index 28ef44e980..80fe86d297 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_CN.md b/README_CN.md index a4ce915ca2..6e9acf15b7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_ES.md b/README_ES.md index 53ef00b81e..6fd9aa3474 100644 --- a/README_ES.md +++ b/README_ES.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_ID.md b/README_ID.md index 8d93bb2447..048143da7b 100644 --- a/README_ID.md +++ b/README_ID.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_JP.md b/README_JP.md index 81e8cefe4c..ed78f15d49 100644 --- a/README_JP.md +++ b/README_JP.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_KR.md b/README_KR.md index 85c227cf6a..6f4aed10c4 100644 --- a/README_KR.md +++ b/README_KR.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_PT-BR.md b/README_PT-BR.md index abd3637168..b624a23fae 100644 --- a/README_PT-BR.md +++ b/README_PT-BR.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_TR.md b/README_TR.md index a729de6764..7080c7f88e 100644 --- a/README_TR.md +++ b/README_TR.md @@ -147,7 +147,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger) (default "list") + -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/internal/tests/integration/fuzz_test.go b/internal/tests/integration/fuzz_test.go index 87fbe21894..7c47c28498 100644 --- a/internal/tests/integration/fuzz_test.go +++ b/internal/tests/integration/fuzz_test.go @@ -5,9 +5,13 @@ package integration_test import ( "fmt" + "io" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" + "strings" "github.com/julienschmidt/httprouter" "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" @@ -36,6 +40,55 @@ var fuzzingTestCases = []integrationCase{ {Path: "fuzz/fuzz-body-params-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}}, {Path: "fuzz/fuzz-body-xml-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 1}}, {Path: "fuzz/fuzz-body-generic-sqli.yaml", TestCase: &genericFuzzTestCase{expectedResults: 4}}, + {Path: "fuzz/fuzz-raw-http-input.yaml", TestCase: &rawHttpInputFuzz{}}, +} + +// rawHttpInputFuzz seeds the base request from a raw HTTP request file, the +// shape users have on hand when there is no spec or captured traffic to feed in. +type rawHttpInputFuzz struct{} + +func (h *rawHttpInputFuzz) Execute(filePath string) error { + router := httprouter.New() + router.POST("/api/login", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "text/html") + var payload map[string]string + if err := json.Unmarshal(body, &payload); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // stand in for a backend that interpolates the value into a query + if strings.Contains(payload["user"], "'") { + _, _ = fmt.Fprint(w, "unrecognized token:") + return + } + _, _ = fmt.Fprintf(w, "welcome %s", payload["user"]) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + target, err := url.Parse(ts.URL) + if err != nil { + return err + } + rawRequest := fmt.Sprintf("POST /api/login HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\n\r\n{\"user\":\"admin\"}", target.Host) + targetPath := filepath.Join(os.TempDir(), "nuclei-raw-http-input.http") + if err := os.WriteFile(targetPath, []byte(rawRequest), 0644); err != nil { + return err + } + defer func() { + _ = os.Remove(targetPath) + }() + + results, err := testutils.RunNucleiWithArgsAndGetResults(debug, "-t", filePath, "-l", targetPath, "-im", "http", "-dast") + if err != nil { + return err + } + return expectResultsCount(results, 1) } type genericFuzzTestCase struct { diff --git a/internal/tests/integration/testdata/fuzz/fuzz-raw-http-input.yaml b/internal/tests/integration/testdata/fuzz/fuzz-raw-http-input.yaml new file mode 100644 index 0000000000..4ecbf3436a --- /dev/null +++ b/internal/tests/integration/testdata/fuzz/fuzz-raw-http-input.yaml @@ -0,0 +1,34 @@ +id: raw-http-input-body-fuzz + +info: + name: fuzzing json body seeded from a raw http request file + author: pdteam + severity: info + description: | + This template fuzzes the json body of a base request seeded from a raw HTTP + request file (-im http), covering the shortest path from a single request to + a DAST scan. + +http: + - pre-condition: + - type: dsl + dsl: + - method == "POST" + - contains(content_type, "application/json") + condition: and + + payloads: + injection: + - "'" + + fuzzing: + - part: body + type: postfix + mode: single + fuzz: + - '{{injection}}' + + matchers: + - type: word + words: + - "unrecognized token:" diff --git a/pkg/input/README.md b/pkg/input/README.md index b5936c5803..e82e5f31b3 100644 --- a/pkg/input/README.md +++ b/pkg/input/README.md @@ -27,3 +27,40 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) This function returns a InputProvider based by appropriately selecting input provider based on the input format (i.e. either list or http) and returns the provider that can handle that input format. + +## Formats + +Fuzzing and DAST take the request shape (method, body, params) from the input, not from CLI flags, so a bare URL is always fuzzed as a `GET`. To fuzz anything else, feed a request-shaped input with `-im`: + +| `-im` | Input | +| --- | --- | +| `list` | urls, domains, ips, cidrs (default) | +| `http` | raw HTTP requests | +| `burp` | Burp Suite xml export | +| `jsonl` | proxify jsonl output | +| `yaml` | proxify yaml multidoc output | +| `openapi` | OpenAPI 3 spec | +| `swagger` | Swagger 2 spec | + +### Raw HTTP requests (`-im http`) + +The shortest path from a single endpoint to a DAST scan, for targets with no spec or captured traffic. Save the request as it appears on the wire, which is what "copy as raw request" in Burp and browser devtools produce: + +``` +POST /api/login HTTP/1.1 +Host: example.com +Content-Type: application/json + +{"user":"admin","pass":"secret"} +``` + +```console +nuclei -l login.http -im http -dast -t fuzzing-templates/ +``` + +Notes: + +- Separate multiple requests with a line starting with `###`, the `.http` file convention. +- The scheme is not part of the request, so it is inferred from the authority. Use an absolute request target (`POST https://example.com/api/login HTTP/1.1`) to state it explicitly. +- The target comes from the `Host` header or from an absolute request target; a request carrying neither is skipped. + diff --git a/pkg/input/formats/raw/raw.go b/pkg/input/formats/raw/raw.go new file mode 100644 index 0000000000..a64c3f904a --- /dev/null +++ b/pkg/input/formats/raw/raw.go @@ -0,0 +1,99 @@ +// Package raw parses input files holding raw HTTP requests, the shape produced +// by "copy as raw request" in Burp and browser devtools and stored in .http +// files. It is the shortest path from a single request to a DAST scan, for +// targets that have no OpenAPI spec or captured traffic to feed in. +package raw + +import ( + "io" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" +) + +// requestSeparator delimits requests when a file carries more than one, +// following the .http file convention +const requestSeparator = "###" + +// RawFormat is a parser for files containing one or more raw HTTP requests +type RawFormat struct { + opts formats.InputFormatOptions +} + +// New creates a new raw HTTP request parser +func New() *RawFormat { + return &RawFormat{} +} + +var _ formats.Format = &RawFormat{} + +// Name returns the name of the format +func (r *RawFormat) Name() string { + return "http" +} + +func (r *RawFormat) SetOptions(options formats.InputFormatOptions) { + r.opts = options +} + +// Parse parses the input and calls the provided callback +// function for each RawRequest it discovers. +func (r *RawFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { + data, err := io.ReadAll(input) + if err != nil { + return errors.Wrap(err, "could not read raw request file") + } + for _, request := range splitRequests(string(data)) { + reqResp, err := types.ParseRawRequest(terminateHeaders(request)) + if err != nil { + gologger.Warning().Msgf("http: could not parse raw request in %s: %s\n", filePath, err) + continue + } + if reqResp.URL.Host == "" { + gologger.Warning().Msgf("http: skipped raw request in %s: no target, add a Host header or an absolute request target\n", filePath) + continue + } + if reqResp.URL.Scheme == "" { + // nothing in the request states the scheme, so settle it the same + // way nuclei settles it for scheme-less list inputs + reqResp.URL.Scheme = utils.DetermineSchemeOrder(reqResp.URL.Host)[0] + } + resultsCb(reqResp) + } + return nil +} + +// splitRequests splits the file contents into individual raw requests +func splitRequests(data string) []string { + var requests []string + var current []string + flush := func() { + if request := strings.Join(current, "\n"); strings.TrimSpace(request) != "" { + requests = append(requests, strings.TrimLeft(request, "\r\n")) + } + current = nil + } + for _, line := range strings.Split(data, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), requestSeparator) { + flush() + continue + } + current = append(current, line) + } + flush() + return requests +} + +// terminateHeaders appends the blank line that separates headers from the body. +// Requests copied out of a browser or Burp routinely end at the last header, +// and the parser needs that terminator to know where the headers stop. +func terminateHeaders(request string) string { + if strings.Contains(request, "\n\r\n") || strings.Contains(request, "\n\n") { + return request + } + return request + "\r\n\r\n" +} diff --git a/pkg/input/formats/raw/raw_test.go b/pkg/input/formats/raw/raw_test.go new file mode 100644 index 0000000000..567f80b39a --- /dev/null +++ b/pkg/input/formats/raw/raw_test.go @@ -0,0 +1,119 @@ +package raw + +import ( + "strings" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/stretchr/testify/require" +) + +func parseAll(t *testing.T, data string) []*types.RequestResponse { + t.Helper() + var got []*types.RequestResponse + err := New().Parse(strings.NewReader(data), func(rr *types.RequestResponse) bool { + got = append(got, rr) + return false + }, "test.http") + require.NoError(t, err) + return got +} + +func TestParseSingleRequest(t *testing.T) { + got := parseAll(t, "POST /api/login HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/json\r\n\r\n{\"user\":\"admin\"}") + + require.Len(t, got, 1) + require.Equal(t, "POST", got[0].Request.Method) + require.Equal(t, "https://example.com/api/login", got[0].URL.String()) + require.Equal(t, `{"user":"admin"}`, got[0].Request.Body) + contentType, _ := got[0].Request.Headers.Get("Content-Type") + require.Equal(t, "application/json", contentType) +} + +func TestParseMultipleRequests(t *testing.T) { + got := parseAll(t, strings.Join([]string{ + "GET /one HTTP/1.1", + "Host: example.com", + "", + "### second request", + "POST /two HTTP/1.1", + "Host: example.com", + "", + "body=1", + }, "\n")) + + require.Len(t, got, 2) + require.Equal(t, "https://example.com/one", got[0].URL.String()) + require.Equal(t, "https://example.com/two", got[1].URL.String()) + require.Equal(t, "body=1", got[1].Request.Body) +} + +// A request pasted out of a browser or Burp commonly ends at the last header. +func TestParseWithoutTrailingBlankLine(t *testing.T) { + got := parseAll(t, "GET /api/users?id=1 HTTP/1.1\r\nHost: example.com") + + require.Len(t, got, 1) + require.Equal(t, "https://example.com/api/users?id=1", got[0].URL.String()) + require.Empty(t, got[0].Request.Body) +} + +// The scheme is not part of an origin-form request, so it is inferred from the +// authority unless the request target states it. +func TestParseResolvesScheme(t *testing.T) { + tests := []struct { + name string + raw string + url string + }{ + {"no port", "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", "https://example.com/"}, + {"port 80", "GET / HTTP/1.1\r\nHost: example.com:80\r\n\r\n", "http://example.com:80/"}, + {"high port", "GET / HTTP/1.1\r\nHost: 127.0.0.1:18080\r\n\r\n", "http://127.0.0.1:18080/"}, + {"absolute target", "GET https://example.com/p HTTP/1.1\r\nHost: example.com\r\n\r\n", "https://example.com/p"}, + {"absolute plain target", "GET http://example.com/p HTTP/1.1\r\nHost: example.com\r\n\r\n", "http://example.com/p"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseAll(t, tt.raw) + require.Len(t, got, 1) + require.Equal(t, tt.url, got[0].URL.String()) + }) + } +} + +// A request with no way to resolve a target is dropped rather than scanned +// against nothing, and it must not take the rest of the file down with it. +func TestParseSkipsRequestsWithoutTarget(t *testing.T) { + got := parseAll(t, strings.Join([]string{ + "GET /no-host HTTP/1.1", + "X-Test: 1", + "", + "###", + "GET /valid HTTP/1.1", + "Host: example.com", + "", + }, "\n")) + + require.Len(t, got, 1) + require.Equal(t, "https://example.com/valid", got[0].URL.String()) +} + +// The callback's return value is unused across every input format, and its two +// call sites in the provider give it opposite meanings, so a request must be +// reported regardless of what the previous one returned. +func TestParseIgnoresCallbackReturn(t *testing.T) { + var count int + err := New().Parse(strings.NewReader("GET /one HTTP/1.1\nHost: example.com\n\n###\nGET /two HTTP/1.1\nHost: example.com\n\n"), func(rr *types.RequestResponse) bool { + count++ + return true + }, "test.http") + + require.NoError(t, err) + require.Equal(t, 2, count) +} + +func TestFormatInterface(t *testing.T) { + var format formats.Format = New() + require.Equal(t, "http", format.Name()) +} diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index ee8cb68093..6b8d476f62 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/raw" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/swagger" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/yaml" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -145,6 +146,7 @@ var providersList = []formats.Format{ yaml.New(), openapi.New(), swagger.New(), + raw.New(), } // SupportedFormats returns the list of supported formats in comma-separated diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go index 9c2cddf688..797fe288f1 100644 --- a/pkg/input/types/http.go +++ b/pkg/input/types/http.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/useragent" "github.com/projectdiscovery/utils/conversion" mapsutil "github.com/projectdiscovery/utils/maps" + stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ) @@ -239,25 +240,19 @@ func ParseRawRequest(raw string) (rr *RequestResponse, err error) { method := parts[0] rr.Request.Method = method - // parse relative url - urlx, err := urlutil.ParseRawRelativePath(parts[1], true) + // the request target is normally an origin-form path, but proxy captures and + // .http files use the absolute form, which already carries the authority + var urlx *urlutil.URL + if stringsutil.HasPrefixAny(parts[1], urlutil.HTTP+"://", urlutil.HTTPS+"://") { + urlx, err = urlutil.ParseAbsoluteURL(parts[1], true) + } else { + urlx, err = urlutil.ParseRawRelativePath(parts[1], true) + } if err != nil { return nil, fmt.Errorf("failed to parse url: %s", err) } rr.URL = *urlx - // parse host line - hostLine, err := protoReader.ReadLine() - if err != nil { - return nil, fmt.Errorf("failed to read host line: %s", err) - } - sep := strings.Index(hostLine, ":") - if sep <= 0 || sep >= len(hostLine)-1 { - return nil, fmt.Errorf("invalid host line: %s", hostLine) - } - hostLine = hostLine[sep+2:] - rr.URL.Host = hostLine - // parse headers rr.Request.Headers = mapsutil.NewOrderedMap[string, string]() for { @@ -269,11 +264,23 @@ func ParseRawRequest(raw string) (rr *RequestResponse, err error) { // end of headers next is body break } - parts := strings.SplitN(line, ":", 2) - if len(parts) != 2 { + key, value, found := strings.Cut(line, ":") + if !found || key == "" { return nil, fmt.Errorf("invalid header line: %s", line) } - rr.Request.Headers.Set(parts[0], parts[1][1:]) + value = strings.TrimSpace(value) + // Host carries the authority rather than request metadata, and callers + // read it off the URL: retryablehttp derives the wire Host from there, + // and keeping it in the header map would expose it to header fuzzing as + // if it were an ordinary header. + if strings.EqualFold(key, "Host") { + // an absolute request target takes precedence over the Host header + if rr.URL.Host == "" { + rr.URL.Host = value + } + continue + } + rr.Request.Headers.Set(key, value) } // parse body diff --git a/pkg/input/types/http_test.go b/pkg/input/types/http_test.go index 8cc44e39cb..c3a7588a46 100644 --- a/pkg/input/types/http_test.go +++ b/pkg/input/types/http_test.go @@ -121,6 +121,117 @@ func TestParseRawRequestBodyTrailingNewlines(t *testing.T) { } } +// Headers used to be parsed positionally: the second line was always taken as +// the Host line and dropped from the header map, and values were read by +// skipping a single byte after the colon. Any request-shaped input (burp, jsonl, +// yaml, openapi) hitting one of these shapes either lost a header silently, +// truncated a value, or crashed the whole scan. +func TestParseRawRequestHeaderParsing(t *testing.T) { + tests := []struct { + name string + raw string + host string + headers map[string]string + }{ + { + name: "host first", + raw: "POST /login HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/json\r\n\r\n{}", + host: "example.com", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "host after other headers", + raw: "POST /login HTTP/1.1\r\nContent-Type: application/json\r\nHost: example.com\r\n\r\n{}", + host: "example.com", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "host absent", + raw: "POST /login HTTP/1.1\r\nContent-Type: application/json\r\n\r\n{}", + host: "", + headers: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "no space after colon", + raw: "GET / HTTP/1.1\r\nHost:example.com\r\nX-Token:abc\r\n\r\n", + host: "example.com", + headers: map[string]string{"X-Token": "abc"}, + }, + { + name: "valueless header", + raw: "GET / HTTP/1.1\r\nHost: example.com\r\nX-Empty:\r\n\r\n", + host: "example.com", + headers: map[string]string{"X-Empty": ""}, + }, + { + name: "lowercase host key", + raw: "GET / HTTP/1.1\r\ncontent-type: text/plain\r\nhost: example.com\r\n\r\n", + host: "example.com", + headers: map[string]string{"content-type": "text/plain"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rr *RequestResponse + var err error + require.NotPanics(t, func() { + rr, err = ParseRawRequest(tt.raw) + }) + require.NoError(t, err) + require.Equal(t, tt.host, rr.URL.Host) + + got := map[string]string{} + rr.Request.Headers.Iterate(func(k, v string) bool { + got[k] = v + return true + }) + require.Equal(t, tt.headers, got) + }) + } +} + +// Requests captured through a proxy carry an absolute request target, which the +// parser used to append to the authority as if it were a path. +func TestParseRawRequestAbsoluteTarget(t *testing.T) { + tests := []struct { + name string + raw string + url string + }{ + { + name: "absolute target", + raw: "GET http://example.com/p?q=1 HTTP/1.1\r\nHost: example.com\r\n\r\n", + url: "http://example.com/p?q=1", + }, + { + name: "absolute target wins over host header", + raw: "GET https://example.com/p HTTP/1.1\r\nHost: proxy.internal\r\n\r\n", + url: "https://example.com/p", + }, + { + name: "origin form keeps host header", + raw: "GET /p HTTP/1.1\r\nHost: example.com\r\n\r\n", + url: "example.com/p", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rr, err := ParseRawRequest(tt.raw) + require.NoError(t, err) + require.Equal(t, tt.url, rr.URL.String()) + }) + } +} + +// A header line without a colon is not a header, and must not be mistaken for +// the start of the body. +func TestParseRawRequestRejectsMalformedHeader(t *testing.T) { + _, err := ParseRawRequest("GET / HTTP/1.1\r\nHost: example.com\r\nnot-a-header\r\n\r\n") + require.Error(t, err) +} + func TestUnmarshalJSON(t *testing.T) { tests := []struct { name string diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go index 602715f432..8dde9edbad 100644 --- a/pkg/utils/http_probe.go +++ b/pkg/utils/http_probe.go @@ -27,8 +27,9 @@ var httpFirstSchemes = []string{ "https", } -// determineSchemeOrder for the input -func determineSchemeOrder(input string) []string { +// DetermineSchemeOrder returns the schemes to try for a scheme-less input, +// most likely first +func DetermineSchemeOrder(input string) []string { if _, port, err := net.SplitHostPort(input); err == nil { // if input has port that is commonly used for HTTP, return http then https if sliceutil.Contains(commonHttpPorts, port) { @@ -51,7 +52,7 @@ func determineSchemeOrder(input string) []string { // If none succeeds, probing is abandoned for such URLs. func ProbeURL(input string, httpxclient *httpx.HTTPX) string { normalizedInput := normalizeProbeInput(input) - schemes := determineSchemeOrder(normalizedInput) + schemes := DetermineSchemeOrder(normalizedInput) for _, scheme := range schemes { formedURL := fmt.Sprintf("%s://%s", scheme, normalizedInput) req, err := httpxclient.NewRequest(http.MethodHead, formedURL) diff --git a/pkg/utils/http_probe_test.go b/pkg/utils/http_probe_test.go index 6e417e3a39..1127c7992c 100644 --- a/pkg/utils/http_probe_test.go +++ b/pkg/utils/http_probe_test.go @@ -30,7 +30,7 @@ func TestDetermineSchemeOrder(t *testing.T) { for _, tc := range tests { t.Run(tc.input, func(t *testing.T) { - actual := determineSchemeOrder(tc.input) + actual := DetermineSchemeOrder(tc.input) require.Equal(t, tc.expected, actual) }) } @@ -52,7 +52,7 @@ func TestDetermineSchemeOrderWithHighPorts(t *testing.T) { for _, tc := range tests { t.Run(tc.input, func(t *testing.T) { - actual := determineSchemeOrder(tc.input) + actual := DetermineSchemeOrder(tc.input) require.Equal(t, tc.expected, actual) }) } @@ -69,7 +69,7 @@ func TestDetermineSchemeOrderAmbiguousIPv6Literal(t *testing.T) { for _, tc := range tests { t.Run(tc.input, func(t *testing.T) { - actual := determineSchemeOrder(normalizeProbeInput(tc.input)) + actual := DetermineSchemeOrder(normalizeProbeInput(tc.input)) require.Equal(t, tc.expected, actual) }) }