main.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

// Article JSON struct
type Article struct {
	TypeOf                 string        `json:"type_of"`
	ID                     int           `json:"id"`
	Title                  string        `json:"title"`
	Description            string        `json:"description"`
	CoverImage             string        `json:"cover_image"`
	PublishedAt            time.Time     `json:"published_at"`
	ReadablePublishDate    string        `json:"readable_publish_date"`
	SocialImage            string        `json:"social_image"`
	TagList                string        `json:"tag_list"`
	Slug                   string        `json:"slug"`
	Path                   string        `json:"path"`
	URL                    string        `json:"url"`
	CanonicalURL           string        `json:"canonical_url"`
	CommentsCount          int           `json:"comments_count"`
	PositiveReactionsCount int           `json:"positive_reactions_count"`
	BodyHTML               string        `json:"body_html"`
	LtagStyle              []interface{} `json:"ltag_style"`
	LtagScript             []interface{} `json:"ltag_script"`
	User                   struct {
		Name            string `json:"name"`
		Username        string `json:"username"`
		TwitterUsername string `json:"twitter_username"`
		GithubUsername  string `json:"github_username"`
		WebsiteURL      string `json:"website_url"`
		ProfileImage    string `json:"profile_image"`
		ProfileImage90  string `json:"profile_image_90"`
	} `json:"user"`
}

// Articles array JSON struct
type Articles []struct {
	TypeOf                 string    `json:"type_of"`
	ID                     int32     `json:"id"`
	Title                  string    `json:"title"`
	Description            string    `json:"description"`
	CoverImage             string    `json:"cover_image"`
	PublishedAt            time.Time `json:"published_at"`
	TagList                []string  `json:"tag_list"`
	Slug                   string    `json:"slug"`
	Path                   string    `json:"path"`
	URL                    string    `json:"url"`
	CanonicalURL           string    `json:"canonical_url"`
	CommentsCount          int       `json:"comments_count"`
	PositiveReactionsCount int       `json:"positive_reactions_count"`
	User                   struct {
		Name            string      `json:"name"`
		Username        string      `json:"username"`
		TwitterUsername string      `json:"twitter_username"`
		GithubUsername  interface{} `json:"github_username"`
		WebsiteURL      string      `json:"website_url"`
		ProfileImage    string      `json:"profile_image"`
		ProfileImage90  string      `json:"profile_image_90"`
	} `json:"user"`
}

// DevtoClient struct
type DevtoClient struct {
	DevtoAPIURL string
	Client      *http.Client
}

// New returns our DevtoClient
func New(apiurl string, client *http.Client) *DevtoClient {
	if client == nil {
		client = http.DefaultClient
	}
	return &DevtoClient{
		apiurl,
		client,
	}
}

// FormatPagedRequest reutnrs *http.Request ready to do() to get one page
func (dtc DevtoClient) FormatPagedRequest(param, paramValue string) (*http.Request, error) {
	URL := dtc.DevtoAPIURL
	if param == "page" && paramValue != "" {
		URL = dtc.DevtoAPIURL + "articles/?" + param + "=" + paramValue
		fmt.Printf("%v\n", URL)
	}
	return http.NewRequest(http.MethodGet, URL, nil)
}

// FormatArticleRequest returns http.Request ready to do() and get an article
func (dtc DevtoClient) FormatArticleRequest(i int32) (*http.Request, error) {
	URL := fmt.Sprintf(dtc.DevtoAPIURL+"articles/%d", i)
	return http.NewRequest(http.MethodGet, URL, nil)
}

func getArticle(dtc *DevtoClient, i int32, wg *sync.WaitGroup) {
	defer wg.Done()
	r, err := dtc.FormatArticleRequest(i)
	if err != nil {
		panic(err)
	}

	resp, err := dtc.Client.Do(r)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}

	//var article Article
	//json.Unmarshal(body, &article)
	//fmt.Printf("%v", article.BodyHTML)

	fileName := fmt.Sprintf("%d.json", i)
	ioutil.WriteFile("./out/"+fileName, body, 0666)
}

func main() {
	dtc := New("https://dev.to/api/", nil)
	doit := true
	c := 1

	for doit {
		req, err := dtc.FormatPagedRequest("page", fmt.Sprintf("%d", c))
		if err != nil {
			panic(err)
		}
		resp, err := dtc.Client.Do(req)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}

		var wg sync.WaitGroup
		var articles Articles
		err = json.Unmarshal(body, &articles)
		if err != nil {
			panic(err)
		}
		wg.Add(len(articles))

		for i := range articles {
			go getArticle(dtc, articles[i].ID, &wg)
		}
		wg.Wait()

		if string(body) != "[]" {
			c++
			continue
		}
		doit = false
	}
}