api, commands, menu: Use linked partitioning instead of offset-based pagination

Also delete offset flag

More here: https://developers.soundcloud.com/blog/offset-pagination-deprecated
and here: https://stackoverflow.com/questions/43940103/how-to-get-more-than-350-favorites-with-soundcloud-api

Fix #17

Author: n10v

README.md

@@ -66,10 +66,6 @@ Also commands may be abbreviated to one symbol length. For example, you can inpu
 
 	$ nehm get 3
 
-#### Download second like and don't add it to iTunes playlist
-
-	$ nehm get -o 1 -i ''
-
 #### Download track from URL
 
 	$ nehm get soundcloud.com/nasa/golden-record-russian-greeting

api/api.go

@@ -6,62 +6,29 @@ package api
 
 import (
 	"encoding/json"
-	"math"
-	"net/url"
+	u "net/url"
 	"strconv"
 
 	"github.com/bogem/nehm/logs"
 	"github.com/bogem/nehm/track"
 )
 
 const (
-	tracksLimit    = 150
+	maxLimit       = 200
 	soundCloudLink = "http://soundcloud.com/"
 )
 
-func Favorites(count, offset uint, uid string) ([]track.Track, error) {
-	requestsCount := float64(count) / float64(tracksLimit)
-	requestsCount = math.Ceil(requestsCount)
-
-	var limit uint
-	var tracks []track.Track
-	params := url.Values{}
-	for i := uint(0); i < uint(requestsCount); i++ {
-		if count < tracksLimit {
-			limit = count
-		} else {
-			limit = tracksLimit
-		}
-		count -= limit
-
-		params.Set("limit", strconv.Itoa(int(limit)))
-		params.Set("offset", strconv.Itoa(int((i*tracksLimit)+offset)))
-
-		bFavs, err := get(formFavoritesURI(uid, params))
-		if err == ErrNotFound {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		var favs []track.Track
-		if err := json.Unmarshal(bFavs, &favs); err != nil {
-			logs.FATAL.Fatalln("could't unmarshal JSON with likes:", err)
-		}
-		tracks = append(tracks, favs...)
-	}
-	return tracks, nil
+func Favorites(limit uint, uid string) ([]track.Track, error) {
+	p := NewPaginator(FormFavoritesURL(limit, uid))
+	return p.NextPage()
 }
 
 func AllFavorites(uid string) ([]track.Track, error) {
-	offset := 0
+	p := NewPaginator(FormFavoritesURL(maxLimit, uid))
 	tracks := make([]track.Track, 0)
 
-	// Run loop while len(fav) != 0.
-	// If len(fav) == 0, then we know, what there are no more favorites by user
-	for {
-		favs, err := Favorites(tracksLimit, uint(offset), uid)
+	for !p.OnLastPage() {
+		favs, err := p.NextPage()
 		tracks = append(tracks, favs...)
 		if err == ErrForbidden {
 			break
@@ -72,8 +39,6 @@ func AllFavorites(uid string) ([]track.Track, error) {
 		if len(favs) == 0 {
 			break
 		}
-
-		offset += tracksLimit
 	}
 
 	return tracks, nil
@@ -84,10 +49,11 @@ type JSONUser struct {
 }
 
 func UID(permalink string) string {
-	params := url.Values{}
+	params := u.Values{}
 	params.Set("url", soundCloudLink+permalink)
+	params.Set("client_id", clientID)
 
-	bUser, err := get(formResolveURI(params))
+	bUser, err := get(formResolveURL(params.Encode()))
 	if err != nil {
 		logs.FATAL.Fatalln("there was a problem by resolving an id of user:", err)
 	}
@@ -100,30 +66,12 @@ func UID(permalink string) string {
 	return strconv.Itoa(jUser.ID)
 }
 
-func Search(query string, limit, offset uint) ([]track.Track, error) {
-	params := url.Values{}
-	params.Set("q", query)
-	params.Set("limit", strconv.Itoa(int(limit)))
-	params.Set("offset", strconv.Itoa(int(offset)))
-
-	bFound, err := get(formSearchURI(params))
-	if err != nil {
-		return nil, err
-	}
-
-	var found []track.Track
-	if err := json.Unmarshal(bFound, &found); err != nil {
-		logs.FATAL.Fatalln("couldn't unmarshal JSON with search results:", err)
-	}
-
-	return found, nil
-}
-
-func TrackFromURI(uri string) []track.Track {
-	params := url.Values{}
-	params.Set("url", uri)
+func TrackFromURL(url string) []track.Track {
+	params := u.Values{}
+	params.Set("url", url)
+	params.Set("client_id", clientID)
 
-	bTrack, err := get(formResolveURI(params))
+	bTrack, err := get(formResolveURL(params.Encode()))
 	if err == ErrForbidden {
 		logs.FATAL.Fatalln("you haven't got any access to this track:", err)
 	}

api/http_client.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"errors"
 	"net/url"
+	"strconv"
 
 	"github.com/bogem/nehm/logs"
 	"github.com/valyala/fasthttp"
@@ -22,44 +23,49 @@ var (
 	ErrForbidden = errors.New("403 - you're not allowed to see these tracks")
 	ErrNotFound  = errors.New("404 - there are no tracks")
 
-	uriBuffer = new(bytes.Buffer)
+	urlBuffer = new(bytes.Buffer)
 )
 
-func formResolveURI(params url.Values) string {
-	params.Set("client_id", clientID)
-
-	uriBuffer.Reset()
-	uriBuffer.WriteString(apiURL)
-	uriBuffer.WriteString("/resolve?")
-	uriBuffer.WriteString(params.Encode())
-	return uriBuffer.String()
+func formResolveURL(query string) string {
+	urlBuffer.Reset()
+	urlBuffer.WriteString(apiURL)
+	urlBuffer.WriteString("/resolve?")
+	urlBuffer.WriteString(query)
+	return urlBuffer.String()
 }
 
-func formSearchURI(params url.Values) string {
-	params.Set("client_id", clientID)
+func FormSearchURL(limit uint, query string) string {
+	params := url.Values{}
+	params.Set("limit", strconv.Itoa(int(limit)))
+	params.Set("linked_partitioning", "1")
+	params.Set("client_id", "11a37feb6ccc034d5975f3f803928a32")
+	params.Set("q", query)
 
-	uriBuffer.Reset()
-	uriBuffer.WriteString(apiURL)
-	uriBuffer.WriteString("/tracks?")
-	uriBuffer.WriteString(params.Encode())
-	return uriBuffer.String()
+	urlBuffer.Reset()
+	urlBuffer.WriteString(apiURL)
+	urlBuffer.WriteString("/tracks?")
+	urlBuffer.WriteString(params.Encode())
+	return urlBuffer.String()
 }
 
-func formFavoritesURI(uid string, params url.Values) string {
-	params.Set("client_id", clientID)
+func FormFavoritesURL(limit uint, uid string) string {
+	params := url.Values{}
+	params.Set("limit", strconv.Itoa(int(limit)))
+	params.Set("linked_partitioning", "1")
+	params.Set("client_id", "11a37feb6ccc034d5975f3f803928a32")
 
-	uriBuffer.Reset()
-	uriBuffer.WriteString(apiURL)
-	uriBuffer.WriteString("/users/")
-	uriBuffer.WriteString(uid)
-	uriBuffer.WriteString("/favorites?")
-	uriBuffer.WriteString(params.Encode())
-	return uriBuffer.String()
+	urlBuffer.Reset()
+	urlBuffer.WriteString(apiURL)
+	urlBuffer.WriteString("/users/")
+	urlBuffer.WriteString(uid)
+	urlBuffer.WriteString("/favorites?")
+	urlBuffer.WriteString(params.Encode())
+	return urlBuffer.String()
 }
 
-func get(uri string) ([]byte, error) {
-	logs.INFO.Println("GET", uri)
-	statusCode, body, err := fasthttp.Get(nil, uri)
+func get(url string) ([]byte, error) {
+	logs.INFO.Println("GET", url)
+	statusCode, body, err := fasthttp.Get(nil, url)
 	if err != nil {
 		return nil, err
 	}

api/paginator.go

@@ -0,0 +1,105 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/bogem/nehm/track"
+)
+
+var (
+	ErrLastPage  = errors.New("current page is last")
+	ErrFirstPage = errors.New("current page is first")
+)
+
+type paginatedResponse struct {
+	Collection []track.Track `json:"collection"`
+	NextHref   string        `json:"next_href"`
+}
+
+type Paginator struct {
+	// currentPage corresponds to index of pages.
+	// If it's -1, the Paginator was only initialized.
+	currentPage int
+
+	nextHref string
+	pages    []string
+}
+
+// NewPaginator returns new paginator.
+// It needs only the url to the first page.
+// More: https://developers.soundcloud.com/blog/offset-pagination-deprecated.
+func NewPaginator(firstPageURL string) *Paginator {
+	return &Paginator{currentPage: -1, nextHref: firstPageURL}
+}
+
+func getPage(url string) (paginatedResponse, error) {
+	pResponse := paginatedResponse{}
+
+	response, err := get(url)
+	if err != nil {
+		return pResponse, err
+	}
+
+	err = json.Unmarshal(response, &pResponse)
+	return pResponse, err
+}
+
+// NextPage returns tracks on the next page and error, if it occured.
+// If p is on last page, it returns ErrLastPage
+func (p *Paginator) NextPage() ([]track.Track, error) {
+	if p.OnLastPage() {
+		return nil, ErrLastPage
+	}
+
+	p.currentPage++
+
+	// If p.pages already has a link to next page, then use it,
+	// else append p.nextHref to p.pages.
+	if len(p.pages) >= p.currentPage+1 {
+		p.nextHref = p.pages[p.currentPage]
+	} else if p.nextHref != "" {
+		p.pages = append(p.pages, p.nextHref)
+	}
+
+	response, err := getPage(p.nextHref)
+	if err != nil {
+		return nil, err
+	}
+	p.nextHref = response.NextHref
+	return response.Collection, nil
+}
+
+// OnLastPage checks, if current page is last.
+func (p Paginator) OnLastPage() bool {
+	return p.nextHref == ""
+}
+
+// PrevPage returns tracks on the previous page and error, if it occured.
+// If p is on first page, it returns ErrFirstPage
+func (p *Paginator) PrevPage() ([]track.Track, error) {
+	if p.OnFirstPage() {
+		return nil, ErrFirstPage
+	}
+
+	p.currentPage--
+
+	var url string
+	if len(p.pages) >= p.currentPage-1 {
+		url = p.pages[p.currentPage]
+	} else {
+		// This should never happen! :)
+		return nil, ErrFirstPage
+	}
+
+	response, err := getPage(url)
+	if err != nil {
+		return nil, err
+	}
+	return response.Collection, nil
+}
+
+// OnFirstPage checks, if current page is first.
+func (p Paginator) OnFirstPage() bool {
+	return p.currentPage <= 0
+}

commands/commands.go

@@ -21,7 +21,7 @@ var rootCmd = listCommand
 
 // Variables used in flags.
 var (
-	limit, offset                       uint
+	limit                               uint
 	dlFolder, itunesPlaylist, permalink string
 	verbose                             bool
 )
@@ -48,10 +48,6 @@ func addLimitFlag(cmd *cobra.Command) {
 	cmd.Flags().UintVarP(&limit, "limit", "l", 9, "count of tracks on each page")
 }
 
-func addOffsetFlag(cmd *cobra.Command) {
-	cmd.Flags().UintVarP(&offset, "offset", "o", 0, "offset relative to first like")
-}
-
 func addPermalinkFlag(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&permalink, "permalink", "p", "", "user's permalink")
 }

commands/get.go

@@ -28,7 +28,6 @@ var (
 func init() {
 	addDlFolderFlag(getCommand)
 	addItunesPlaylistFlag(getCommand)
-	addOffsetFlag(getCommand)
 	addPermalinkFlag(getCommand)
 }
 
@@ -59,14 +58,13 @@ func getTracks(cmd *cobra.Command, args []string) {
 
 func getLastTracks(count uint) ([]track.Track, error) {
 	logs.FEEDBACK.Println("Getting ID of user")
-	uid := api.UID(config.Get("permalink"))
-	return api.Favorites(count, offset, uid)
+	return api.Favorites(count, api.UID(config.Get("permalink")))
 }
 
 func isSoundCloudURL(url string) bool {
 	return strings.Contains(url, "soundcloud.com")
 }
 
 func getTrackFromURL(url string) []track.Track {
-	return api.TrackFromURI(url)
+	return api.TrackFromURL(url)
 }