-
-
Notifications
You must be signed in to change notification settings - Fork 51
Home
How to do some common and uncommon tasks with requests!
var s string
err := requests.
URL("http://example.com").
ToString(&s).
Fetch(context.Background())
err := requests.
URL("https://postman-echo.com/post").
BodyBytes([]byte(`hello, world`)).
ContentType("text/plain").
Fetch(context.Background())
var data SomeDataType
err := requests.
URL("https://example.com/my-json").
ToJSON(&data).
Fetch(context.Background())
body := MyRequestType{}
var resp MyResponseType
err := requests.
URL("https://example.com/my-json").
BodyJSON(&body).
ToJSON(&resp).
Fetch(context.Background())
// Set headers
var headers postman
err := requests.
URL("https://postman-echo.com/get").
UserAgent("bond/james-bond").
ContentType("secret").
Header("martini", "shaken").
Fetch(context.Background())
var params postman
err := requests.
URL("https://example.com/get?a=1&b=2").
Param("b", "3").
Param("c", "4").
Fetch(context.Background())
// URL is https://example.com/get?a=1&b=3&c=4
// Save a file
err := requests.
URL("http://example.com").
ToFile("myfile.txt").
Fetch(context.Background())
// Send a file
err := requests.
URL("http://example.com").
BodyFile("myfile.txt").
ContentType("text/plain").
Fetch(context.Background())
TODO
More examples TODO
Write a ResponseHandler that outputs a progress bar for downloads using schollz/progressbar:
func withProgbar(dstFile string) requests.ResponseHandler {
return func(res *http.Response) error {
absFile, err := filepath.Abs(dstFile)
if err != nil {
return err
}
dir, file := filepath.Split(absFile)
_ = os.MkdirAll(dir, 0744)
tmpFile, err := os.CreateTemp(dir, ".*."+file)
if err != nil {
return err
}
defer os.Remove(tmpFile.Name())
defer tmpFile.Close()
fmt.Println("downloading to", tmpFile.Name())
bar := progressbar.DefaultBytes(
res.ContentLength,
fmt.Sprintf("downloading %s", file),
)
if _, err = io.Copy(
io.MultiWriter(bar, tmpFile),
res.Body,
); err != nil {
return err
}
if err = tmpFile.Close(); err != nil {
return err
}
fmt.Println("download succeeded")
if err = os.Rename(tmpFile.Name(), absFile); err != nil {
return err
}
fmt.Println("moved to", absFile)
return nil
}
You can do almost anything by writing a requests.RoundTripFunc
. Remember that the http.RoundTripper
interface requires that you copy requests before modifying them. Other than that, the sky is the limit.
See https://pkg.go.dev/github.com/philippta/trip for some examples of custom transports that are compatible with http.RoundTripper
.
By setting tls.Config.RootCAs
to an x509.CertPool
that you have created, you can ensure that your http.Transport
will only connect to the servers you have authorized.
pem := []byte(...)
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(pem)
tlsconfig := tls.Config{
RootCAs: pool,
}
transport := http.Transport{
ForceAttemptHTTP2: true,
TLSClientConfig: &tlsconfig,
}
var s string
err := requests.
URL("https://self-signed.badssl.com").
Transport(&transport).
ToString(&s).
Fetch(context.Background())
By default, requests uses the canonical form for all headers. The HTTP protocol is specified to be case insensitive. However, certain non-compliant servers may require you to use a non-canonical capitalization for a header. This can be done by using a custom transport to modify the headers before they are sent.
basetransport := http.DefaultTransport
lowercaseHeaders := requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
// Custom transports should clone a request before modifying it
req2 := *req
req2.Header = make(http.Header, len(req.Header))
for k, v := range req.Header {
// Make each header key in the clone request lowercase
req2.Header[strings.ToLower(k)] = v
}
// Send clone request using the real transport
return basetransport.RoundTrip(&req2)
})
rb := requests.
URL("https://example.com").
Transport(lowercaseHeaders)
Ensure that a request has a fixed Content-Length for the server by writing body to buffer before sending.
func BufferedBodyTransport(rt http.RoundTripper) http.RoundTripper {
if rt == nil {
rt = http.DefaultTransport
}
return requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
var buf bytes.Buffer
if _, err = io.Copy(&buf, req.Body); err != nil {
return
}
// shallow copy req before modifying
req2 := *req
data := buf.Bytes()
req2.Body = io.NopCloser(bytes.NewReader(data))
req2.ContentLength = int64(len(data))
req2.GetBody = func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(data)), nil
}
return rt.RoundTrip(&req2)
})
}
var s string
err := requests.
URL("http://example.com").
BodyReader(MyXMLReader()).
Transport(BufferedBodyTransport).
ToString(&s).
Fetch(ctx)
ctx := context.Background()
cancelled := errors.New("context was cancelled")
l := rate.NewLimiter(15.0/60, 15)
func RateLimitTransport(rt http.RoundTripper) http.RoundTripper {
if rt == nil {
rt = http.DefaultTransport
}
return requests.RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
if err := l.Wait(ctx); err != nil {
return nil, cancelled
}
return rt.RoundTrip(req)
})
}
var s string
err := requests.
URL("http://example.com").
Transport(RateLimitTransport(nil)).
ToString(&s).
Fetch(ctx)
For automatic retries, use a standard library compatible transport, such as github.com/ybbus/httpretry, github.com/PuerkitoBio/rehttp, or github.com/philippta/trip:
// import "github.com/ybbus/httpretry"
cl := httpretry.NewDefaultClient()
var s string
err := requests.
URL("http://example.com/flakey").
Client(cl).
ToString(&s).
Fetch(ctx)