Merge pull request #42 from twitchdev/feature/autopagination

Adding autopaginate feature

Author: lleadbet

cmd/api.go

@@ -16,6 +16,7 @@ import (
 var queryParameters []string
 var body string
 var prettyPrint bool
+var autoPaginate bool
 
 var apiCmd = &cobra.Command{
 	Use:   "api",
@@ -71,20 +72,23 @@ func init() {
 	apiCmd.PersistentFlags().MarkHidden("pretty-print")
 
 	apiCmd.PersistentFlags().BoolVarP(&prettyPrint, "unformatted", "u", false, "Whether to have API requests come back unformatted/non-prettyprinted. Default is false.")
+
+	getCmd.PersistentFlags().BoolVarP(&autoPaginate, "autopaginate", "P", false, "Whether to have API requests automatically paginate. Default is false.")
+
 }
 
 func cmdRun(cmd *cobra.Command, args []string) {
 	if len(args) == 0 {
 		cmd.Help()
 		return
 	} else if len(args) == 1 && args[0][:1] == "/" {
-		api.NewRequest(cmd.Name(), args[0], queryParameters, []byte(body), !prettyPrint)
+		api.NewRequest(cmd.Name(), args[0], queryParameters, []byte(body), !prettyPrint, autoPaginate)
 		return
 	}
 	if body != "" && body[:1] == "@" {
 		body = getBodyFromFile(body[1:])
 	}
-	api.NewRequest(cmd.Name(), "/"+strings.Join(args[:], "/"), queryParameters, []byte(body), !prettyPrint)
+	api.NewRequest(cmd.Name(), "/"+strings.Join(args[:], "/"), queryParameters, []byte(body), !prettyPrint, autoPaginate)
 }
 
 func getBodyFromFile(filename string) string {

docs/api.md

@@ -29,10 +29,11 @@ Allows the user to make GET calls to endpoints on Helix. Requires a logged in to
 
 **Flags**
 
-| Flag             | Shorthand | Description                                                                                                   | Example              | Required? (Y/N) |
-|------------------|-----------|---------------------------------------------------------------------------------------------------------------|----------------------|-----------------|
-| `--query-param`  | `-q`      | Query parameters for the endpoint in `key=value` format. Multiple can be entered to give multiple parameters. | `get -q login=ninja` | N               |
-| `--unformatted`  | `-u`      | Whether to return unformatted responses. Default is `false`.                                                  | `get -u`             | N               |
+| Flag             | Shorthand | Description                                                                                                                                                         | Example              | Required? (Y/N) |
+|------------------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|-----------------|
+| `--query-param`  | `-q`      | Query parameters for the endpoint in `key=value` format. Multiple can be entered to give multiple parameters.                                                       | `get -q login=ninja` | N               |
+| `--unformatted`  | `-u`      | Whether to return unformatted responses. Default is `false`.                                                                                                        | `get -u`             | N               |
+| `--autopaginate` | `-P`      | Whether to autopaginate the response from Twitch **WARNING** This flag can cause extremely large payloads and cause issues with some terminals. Default is `false`. | `get -P`             | N               |
 
 **Examples**
 

internal/api/api.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/twitchdev/twitch-cli/internal/login"
+	"github.com/twitchdev/twitch-cli/internal/models"
 	"github.com/twitchdev/twitch-cli/internal/util"
 
 	"github.com/TylerBrock/colorjson"
@@ -29,7 +30,11 @@ type clientInformation struct {
 }
 
 // NewRequest is used to request data from the Twitch API using a HTTP GET request- this function is a wrapper for the apiRequest function that handles the network call
-func NewRequest(method string, path string, queryParameters []string, body []byte, prettyPrint bool) {
+func NewRequest(method string, path string, queryParameters []string, body []byte, prettyPrint bool, autopaginate bool) {
+	var data models.APIResponse
+	var err error
+	var cursor string
+
 	client, err := GetClientInformation()
 
 	if viper.GetString("BASE_URL") != "" {
@@ -40,37 +45,94 @@ func NewRequest(method string, path string, queryParameters []string, body []byt
 		fmt.Println("Error fetching client information", err.Error())
 	}
 
-	Parameters := url.Values{}
+	for {
+		var apiResponse models.APIResponse
+
+		u, err := url.Parse(baseURL + path)
+		if err != nil {
+			fmt.Printf("Error getting url: %v", err)
+			return
+		}
 
-	if queryParameters != nil {
-		path += "?"
+		q := u.Query()
 		for _, param := range queryParameters {
 			value := strings.Split(param, "=")
-			Parameters.Add(value[0], value[1])
+			q.Add(value[0], value[1])
+		}
+
+		if cursor != "" {
+			q.Set("after", cursor)
+		}
+
+		if autopaginate == true {
+			first := "100"
+			// since channel points custom rewards endpoints only support 50, capping that here
+			if strings.Contains(u.String(), "custom_rewards") {
+				first = "50"
+			}
+
+			q.Set("first", first)
+		}
+
+		u.RawQuery = q.Encode()
+
+		resp, err := apiRequest(strings.ToUpper(method), u.String(), body, apiRequestParameters{
+			ClientID: client.ClientID,
+			Token:    client.Token,
+		})
+		if err != nil {
+			fmt.Printf("Error reading body: %v", err)
+			return
+		}
+
+		if resp.StatusCode == http.StatusNoContent {
+			fmt.Println("Endpoint responded with status 204")
+			return
+		}
+
+		err = json.Unmarshal(resp.Body, &apiResponse)
+		if err != nil {
+			fmt.Printf("Error unmarshalling body: %v", err)
+			return
 		}
-		path += Parameters.Encode()
+
+		if resp.StatusCode > 299 || resp.StatusCode < 200 {
+			data = apiResponse
+			break
+		}
+
+		data.Data = append(data.Data, apiResponse.Data...)
+
+		if autopaginate == false {
+			data.Pagination.Cursor = apiResponse.Pagination.Cursor
+			break
+		}
+
+		if apiResponse.Pagination.Cursor == "" {
+			break
+		}
+
+		if apiResponse.Pagination.Cursor == cursor {
+			break
+		}
+		cursor = apiResponse.Pagination.Cursor
+
 	}
-	resp, err := apiRequest(strings.ToUpper(method), baseURL+path, body, apiRequestParameters{
-		ClientID: client.ClientID,
-		Token:    client.Token,
-	})
-	if err != nil {
-		fmt.Printf("Error reading body: %v", err)
-		return
+
+	// handle json marshalling better; returns empty slice vs. null
+	if len(data.Data) == 0 && data.Error == "" {
+		data.Data = make([]interface{}, 0)
 	}
 
-	if resp.StatusCode == http.StatusNoContent {
-		fmt.Println("Endpoint responded with status 204")
+	d, err := json.Marshal(data)
+	if err != nil {
+		log.Printf("Error marshalling json: %v", err)
 		return
 	}
 
 	if prettyPrint == true {
 		var obj map[string]interface{}
-		if err := json.Unmarshal(resp.Body, &obj); err != nil {
-			fmt.Printf("Error pretty-printing body: %v", err)
-			return
-		}
-
+		json.Unmarshal(d, &obj)
 		// since Command Prompt/Powershell don't support coloring, will pretty print without colors
 		if runtime.GOOS == "windows" {
 			s, _ := json.MarshalIndent(obj, "", "  ")
@@ -81,12 +143,16 @@ func NewRequest(method string, path string, queryParameters []string, body []byt
 		f := colorjson.NewFormatter()
 		f.Indent = 2
 		f.KeyColor = color.New(color.FgBlue).Add(color.Bold)
-		s, _ := f.Marshal(obj)
-
+		s, err := f.Marshal(obj)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
 		fmt.Println(string(s))
 		return
 	}
-	fmt.Println(string(resp.Body))
+
+	fmt.Println(string(d))
 	return
 }
 

internal/api/api_request.go

@@ -6,10 +6,10 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"time"
 
 	"github.com/twitchdev/twitch-cli/internal/request"
+	"golang.org/x/time/rate"
 )
 
 type apiRequestParameters struct {
@@ -26,14 +26,14 @@ func apiRequest(method string, url string, payload []byte, p apiRequestParameter
 
 	req.Header.Set("Client-ID", p.ClientID)
 	req.Header.Set("Content-Type", "application/json")
+	rl := rate.NewLimiter(rate.Every(time.Minute), 800)
+
+	client := NewClient(rl)
 
 	if p.Token != "" {
 		req.Header.Set("Authorization", "Bearer "+p.Token)
 	}
 
-	client := &http.Client{
-		Timeout: time.Second * 10,
-	}
 	resp, err := client.Do(req)
 	if err != nil {
 		fmt.Printf("Error reading body: %v", err)

internal/api/api_test.go

@@ -39,8 +39,8 @@ func TestNewRequest(t *testing.T) {
 	viper.Set("accesstoken", "4567")
 	viper.Set("refreshtoken", "123")
 
-	NewRequest("POST", "", []string{"test=1", "test=2"}, nil, true)
-	NewRequest("POST", "", []string{"test=1", "test=2"}, nil, false)
+	NewRequest("POST", "", []string{"test=1", "test=2"}, nil, true, false)
+	NewRequest("POST", "", []string{"test=1", "test=2"}, nil, false, true)
 }
 
 func TestValidOptions(t *testing.T) {

internal/api/client.go

@@ -0,0 +1,39 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+type RLClient struct {
+	client      *http.Client
+	RateLimiter *rate.Limiter
+}
+
+func (c *RLClient) Do(req *http.Request) (*http.Response, error) {
+	err := c.RateLimiter.Wait(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+func NewClient(l *rate.Limiter) *RLClient {
+	client := http.Client{
+		Timeout: time.Second * 10,
+	}
+
+	return &RLClient{
+		client:      &client,
+		RateLimiter: l,
+	}
+}

internal/api/client_test.go

@@ -0,0 +1,43 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package api
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/twitchdev/twitch-cli/internal/util"
+	"golang.org/x/time/rate"
+)
+
+func TestNewClient(t *testing.T) {
+	a := util.SetupTestEnv(t)
+
+	var ok = "{\"status\":\"ok\"}"
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(ok))
+
+		_, err := ioutil.ReadAll(r.Body)
+		a.Nil(err)
+
+	}))
+
+	rl := rate.NewLimiter(rate.Every(time.Minute), 800)
+	c := NewClient(rl)
+
+	req, _ := http.NewRequest(http.MethodGet, ts.URL, nil)
+	resp, err := c.Do(req)
+	a.Nil(err)
+
+	body, err := ioutil.ReadAll(resp.Body)
+	defer resp.Body.Close()
+	a.Equal(ok, string(body), "Body mismatch")
+
+	req, _ = http.NewRequest(http.MethodGet, "potato", nil)
+	resp, err = c.Do(req)
+	a.NotNil(err)
+}

internal/models/api.go

@@ -3,5 +3,11 @@
 package models
 
 type APIResponse struct {
-	Data []interface{} `json:"data"`
+	Data       []interface{} `json:"data,omitempty"`
+	Pagination struct {
+		Cursor string `json:"cursor,omitempty"`
+	} `json:"pagination,omitempty"`
+	Error   string `json:"error,omitempty"`
+	Status  int    `json:"status,omitempty"`
+	Message string `json:"message,omitempty"`
 }