Home

Cookbook

How to do some common and uncommon tasks with requests!

Basics

GET response body as a string

var s string
err := requests.
	URL("http://example.com").
	ToString(&s).
	Fetch(context.Background())

POST raw bytes

err := requests.
	URL("https://postman-echo.com/post").
	BodyBytes([]byte(`hello, world`)).
	ContentType("text/plain").
	Fetch(context.Background())

GET JSON object

var data SomeDataType
err := requests.
	URL("https://example.com/my-json").
	ToJSON(&data).
	Fetch(context.Background())

POST JSON and parse JSON response

body := MyRequestType{}
var resp MyResponseType
err := requests.
	URL("https://example.com/my-json").
	BodyJSON(&body).
	ToJSON(&resp).
	Fetch(context.Background())

Custom headers

// Set headers
var headers postman
err := requests.
	URL("https://postman-echo.com/get").
	UserAgent("bond/james-bond").
	ContentType("secret").
	Header("martini", "shaken").
	Fetch(context.Background())

Query parameters

var params postman
err := requests.
	URL("https://example.com/get?a=1&b=2").
	Param("b", "3").
	Param("c", "4").
	Fetch(context.Background())
	// URL is https://example.com/get?a=1&b=3&c=4

Send or save a file

// Save a file
err := requests.
	URL("http://example.com").
	ToFile("myfile.txt").
	Fetch(context.Background())

// Send a file
err := requests.
	URL("http://example.com").
	BodyFile("myfile.txt").
	ContentType("text/plain").
	Fetch(context.Background())

Advanced techniques

Write your own BodyGetter or Config

TODO

Write your own validators/handlers as a ResponseHandler

More examples TODO

Create a progress bar handler

Write a ResponseHandler that outputs a progress bar for downloads using schollz/progressbar:

func withProgbar(dstFile string) requests.ResponseHandler {
	return func(res *http.Response) error {
		absFile, err := filepath.Abs(dstFile)
		if err != nil {
			return err
		}
		dir, file := filepath.Split(absFile)
		_ = os.MkdirAll(dir, 0744)

		tmpFile, err := os.CreateTemp(dir, ".*."+file)
		if err != nil {
			return err
		}
		defer os.Remove(tmpFile.Name())
		defer tmpFile.Close()

		fmt.Println("downloading to", tmpFile.Name())

		bar := progressbar.DefaultBytes(
			res.ContentLength,
			fmt.Sprintf("downloading %s", file),
		)
		if _, err = io.Copy(
			io.MultiWriter(bar, tmpFile),
			res.Body,
		); err != nil {
			return err
		}
		if err = tmpFile.Close(); err != nil {
			return err
		}
		fmt.Println("download succeeded")
		if err = os.Rename(tmpFile.Name(), absFile); err != nil {
			return err
		}
		fmt.Println("moved to", absFile)
		return nil
}

Create and use custom transports

You can do almost anything by writing a requests.RoundTripFunc. Remember that the http.RoundTripper interface requires that you copy requests before modifying them. Other than that, the sky is the limit.

See https://pkg.go.dev/github.com/philippta/trip for some examples of custom transports that are compatible with http.RoundTripper.

Validate certificates

By setting tls.Config.RootCAs to an x509.CertPool that you have created, you can ensure that your http.Transport will only connect to the servers you have authorized.

pem := []byte(...)
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(pem)
tlsconfig := tls.Config{
	RootCAs: pool,
}
transport := http.Transport{
	ForceAttemptHTTP2: true,
	TLSClientConfig:   &tlsconfig,
}
var s string
err := requests.
	URL("https://self-signed.badssl.com").
	Transport(&transport).
	ToString(&s).
	Fetch(context.Background())

Non-canonical header capitalization

By default, requests uses the canonical form for all headers. The HTTP protocol is specified to be case insensitive. However, certain non-compliant servers may require you to use a non-canonical capitalization for a header. This can be done by using a custom transport to modify the headers before they are sent.

basetransport  := http.DefaultTransport
lowercaseHeaders := requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
	// Custom transports should clone a request before modifying it
	req2 := *req
	req2.Header = make(http.Header, len(req.Header))
	for k, v := range req.Header {
		// Make each header key in the clone request lowercase
		req2.Header[strings.ToLower(k)] = v
	}
	// Send clone request using the real transport
	return basetransport.RoundTrip(&req2)
})
rb := requests.
	URL("https://example.com").
	Transport(lowercaseHeaders)

Always set Content-Length

Ensure that a request has a fixed Content-Length for the server by writing body to buffer before sending.

func BufferedBodyTransport(rt http.RoundTripper) http.RoundTripper {
	if rt == nil {
		rt = http.DefaultTransport
	}
	return requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
		var buf bytes.Buffer
		if _, err = io.Copy(&buf, req.Body); err != nil {
			return
		}
		// shallow copy req before modifying
		req2 := *req
		data := buf.Bytes()
		req2.Body = io.NopCloser(bytes.NewReader(data))
		req2.ContentLength = int64(len(data))
		req2.GetBody = func() (io.ReadCloser, error) {
			return io.NopCloser(bytes.NewReader(data)), nil
		}

		return rt.RoundTrip(&req2)
	})
}

var s string
err := requests.
	URL("http://example.com").
	BodyReader(MyXMLReader()).
	Transport(BufferedBodyTransport).
	ToString(&s).
	Fetch(ctx)

Basic rate limiting

ctx := context.Background()
cancelled := errors.New("context was cancelled")
l := rate.NewLimiter(15.0/60, 15)
func RateLimitTransport(rt http.RoundTripper) http.RoundTripper {
	if rt == nil {
		rt = http.DefaultTransport
	}
	return requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
		if err := l.Wait(ctx); err != nil {
			return nil, cancelled
		}

		return rt.RoundTrip(req)
	})
}

var s string
err := requests.
	URL("http://example.com").
	Transport(RateLimitTransport(nil)).
	ToString(&s).
	Fetch(ctx)

Retries

For automatic retries, use a standard library compatible transport, such as github.com/ybbus/httpretry, github.com/PuerkitoBio/rehttp, or github.com/philippta/trip:

// import "github.com/ybbus/httpretry"

cl := httpretry.NewDefaultClient()

var s string
err := requests.
	URL("http://example.com/flakey").
	Client(cl).
	ToString(&s).
	Fetch(ctx)