♻️ Refactored source code, packed providers.json

Author: toby3d

a_oembed-packr.go

@@ -0,0 +1,33 @@
+package oembed
+
+import (
+	"fmt"
+
+	"golang.org/x/xerrors"
+)
+
+// Error represent a complex error
+type Error struct {
+	Message string
+	URL     string
+	Details xerrors.Frame
+}
+
+// Error returns a string formatted error
+func (e Error) Error() string {
+	return fmt.Sprint(e)
+}
+
+// Format implements fmt.Formatter method
+func (e Error) Format(f fmt.State, c rune) {
+	xerrors.FormatError(e, f, c)
+}
+
+// FormatError implements xerrors.Formatter method
+func (e Error) FormatError(p xerrors.Printer) error {
+	p.Printf("ERROR: %d [url:%s]", e.Message, e.URL)
+	if p.Detail() {
+		e.Details.Format(p)
+	}
+	return nil
+}

errors.go

@@ -8,18 +8,19 @@ import (
 	template "github.com/valyala/fasttemplate"
 )
 
+// Params represent a optional parameters for Extract method.
 type Params struct {
 	MaxWidth  int
 	MaxHeight int
 }
 
-func fetchEmbed(url string, provider providerCandidate, params *Params) (*Response, error) {
-	resourceUrl := provider.URL
-	resourceUrl = template.ExecuteString(resourceUrl, "{", "}", map[string]interface{}{"format": "json"})
+func fetchEmbed(url string, provider *Provider, params *Params) (*OEmbed, error) {
+	resourceURL := provider.Endpoints[0].URL
+	resourceURL = template.ExecuteString(resourceURL, "{", "}", map[string]interface{}{"format": "json"})
 
 	link := http.AcquireURI()
 	defer http.ReleaseURI(link)
-	link.Update(resourceUrl)
+	link.Update(resourceURL)
 	qa := link.QueryArgs()
 	qa.Add("format", "json")
 	qa.Add("url", url)
@@ -40,14 +41,20 @@ func fetchEmbed(url string, provider providerCandidate, params *Params) (*Respon
 	defer http.ReleaseResponse(resp)
 
 	if err := http.Do(req, resp); err != nil {
-		return nil, err
+		return nil, Error{
+			Message: err.Error(),
+			URL:     url,
+		}
 	}
 
-	var response Response
-	if err := json.Unmarshal(resp.Body(), &response); err != nil {
-		return nil, err
+	var oEmbed OEmbed
+	if err := json.UnmarshalFast(resp.Body(), &oEmbed); err != nil {
+		return nil, Error{
+			Message: err.Error(),
+			URL:     url,
+		}
 	}
-	response.ProviderName = provider.ProviderName
-	response.ProviderURL = provider.ProviderURL
-	return &response, nil
+	oEmbed.ProviderName = provider.Name
+	oEmbed.ProviderURL = provider.URL
+	return &oEmbed, nil
 }

fetch_embed.go

@@ -11,10 +11,10 @@ func TestFetchEmbed(t *testing.T) {
 	t.Run("valid", func(t *testing.T) {
 		resp, err := fetchEmbed(
 			"https://www.youtube.com/watch?v=8jPQjjsBbIc",
-			makeCandidate(Provider{
+			&Provider{
 				Name: "YouTube",
 				URL:  "https://www.youtube.com/",
-				Endpoints: []Endpoint{Endpoint{
+				Endpoints: []Endpoint{{
 					Schemes: []string{
 						"https://*.youtube.com/watch*",
 						"https://*.youtube.com/v/*\"",
@@ -23,7 +23,7 @@ func TestFetchEmbed(t *testing.T) {
 					URL:       "https://www.youtube.com/oembed",
 					Discovery: true,
 				}},
-			}),
+			},
 			&Params{
 				MaxWidth:  250,
 				MaxHeight: 250,
@@ -34,7 +34,6 @@ func TestFetchEmbed(t *testing.T) {
 	})
 	t.Run("invalid", func(t *testing.T) {
 		for _, url := range []string{
-			"",
 			"htt:/abc.com/failed-none-sense",
 			"https://abc.com/failed-none-sense",
 			"http://badcom/146753785",
@@ -45,12 +44,12 @@ func TestFetchEmbed(t *testing.T) {
 		} {
 			url := url
 			t.Run(url, func(t *testing.T) {
-				c := findProvider(url)
-				if c == nil {
-					c = new(providerCandidate)
+				provider := findProvider(url)
+				if provider == nil {
+					provider = &Provider{Endpoints: []Endpoint{Endpoint{}}}
 				}
 
-				_, err := fetchEmbed(url, *c, nil)
+				_, err := fetchEmbed(url, provider, nil)
 				assert.Error(err)
 			})
 		}

fetch_embed_test.go

@@ -44,26 +44,34 @@ func makeCandidate(p Provider) providerCandidate {
 
 }
 
-func findProvider(url string) *providerCandidate {
-	var candidates []providerCandidate
-	for _, provider := range providersList {
+func findProvider(url string) *Provider {
+	var candidates []Provider
+	for _, provider := range Providers {
 		provider := provider
-		candidate := makeCandidate(provider)
-		if len(candidate.Schemes) == 0 {
-			if !strings.Contains(url, candidate.Domain) {
+
+		endpoint := provider.Endpoints[0]
+		domain := getHostname(endpoint.URL)
+		if domain != "" {
+			domain = strings.TrimPrefix(domain, "www.")
+		} else {
+			domain = ""
+		}
+
+		if len(endpoint.Schemes) == 0 {
+			if !strings.Contains(url, domain) {
 				continue
 			}
-			candidates = append(candidates, candidate)
+			candidates = append(candidates, provider)
 			continue
 		}
-		for _, scheme := range candidate.Schemes {
+		for _, scheme := range endpoint.Schemes {
 			scheme := scheme
 			reg := regexp.MustCompile(strings.Replace(scheme, "*", "(.*)", -1))
 			if !reg.MatchString(url) {
 				continue
 			}
 
-			candidates = append(candidates, candidate)
+			candidates = append(candidates, provider)
 			break
 		}
 	}

find_provider.go

@@ -17,6 +17,7 @@ func TestGetHostname(t *testing.T) {
 		"http://":                  "",
 		"":                         "",
 	} {
+		k, v := k, v
 		t.Run(k, func(t *testing.T) { assert.Equal(v, getHostname(k)) })
 	}
 }
@@ -26,7 +27,7 @@ func TestMakeCandidate(t *testing.T) {
 		Name: "YouTube",
 		URL:  "https://www.youtube.com/",
 		Endpoints: []Endpoint{
-			Endpoint{
+			{
 				Schemes: []string{
 					"https://*.youtube.com/watch*",
 					"https://*.youtube.com/v/*\"",

find_provider_test.go

@@ -1,24 +1,35 @@
 package oembed
 
-import (
-	"golang.org/x/xerrors"
-)
+import "golang.org/x/xerrors"
 
-var (
-	ErrInvalidInputURL = xerrors.New("invalid input url")
-	ErrNoProviderFound = xerrors.New("no provider found with given url")
-)
-
-func Extract(url string, params *Params) (*Response, error) {
+// Extract try fetch oEmbed object for input url with params (if represent).
+// Return OEmbed if success.
+func Extract(url string, params *Params) (*OEmbed, error) {
 	if !isValidURL(url) {
-		return nil, ErrInvalidInputURL
+		return nil, Error{
+			Message: "invalid input url",
+			URL:     url,
+		}
+	}
+	if provider := findProvider(url); provider != nil {
+		resp, err := fetchEmbed(url, provider, params)
+		if err != nil {
+			return nil, Error{
+				Message: err.Error(),
+				URL:     url,
+				Details: xerrors.Caller(1),
+			}
+		}
+		return resp, nil
 	}
-	if c := findProvider(url); c != nil {
-		return fetchEmbed(url, *c, params)
+
+	return nil, Error{
+		Message: "no provider found with given url",
+		URL:     url,
 	}
-	return nil, ErrNoProviderFound
 }
 
+// HasProvider checks what input url has oEmbed provider
 func HasProvider(url string) bool {
 	return findProvider(url) != nil
 }

oembed.go

@@ -1,46 +1,42 @@
 package oembed
 
 import (
-	"io/ioutil"
 	"log"
-	"os"
-	"path/filepath"
+	"time"
 
+	"github.com/gobuffalo/packr"
 	json "github.com/pquerna/ffjson/ffjson"
 	http "github.com/valyala/fasthttp"
 )
 
-const source string = "https://oembed.com/providers.json"
+// SourceURL is a official url of supported providers list
+const SourceURL string = "https://oembed.com/providers.json"
 
-var (
-	providersList []Provider
+// Providers contains all default (or new synced) providers
+var Providers []Provider //nolint:gochecknoglobals
 
-	target = filepath.Join(
-		os.Getenv("GOPATH"), "src", "gitlab.com", "toby3d", "oembed", "assets", "providers.json",
-	)
-)
-
-func init() {
-	if err := fetch(source, target); err != nil {
+func init() { //nolint:gochecknoinits
+	if err := Sync(SourceURL); err != nil {
 		panic(err)
 	}
 }
 
-func fetch(url, target string) error {
-	status, src, err := http.Get(nil, url)
+// Sync try update Providers variable via request and parsing body of sourceURL
+func Sync(sourceURL string) error {
+	status, src, err := http.GetTimeout(nil, sourceURL, 2*time.Second)
 	if err != nil || status != http.StatusOK {
-		if src, err = ioutil.ReadFile(target); err != nil {
-			return err
+		if src, err = packr.NewBox("./assets").Find("providers.json"); err != nil {
+			return Error{
+				Message: err.Error(),
+				URL:     sourceURL,
+			}
 		}
 	}
 
-	if err := json.Unmarshal(src, &providersList); err != nil {
-		return err
-	}
-
-	if status == http.StatusOK {
-		if err = ioutil.WriteFile(target, src, os.ModePerm); err != nil {
-			return err
+	if err = json.Unmarshal(src, &Providers); err != nil {
+		return Error{
+			Message: err.Error(),
+			URL:     sourceURL,
 		}
 	}
 

sync.go

@@ -0,0 +1,24 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSync(t *testing.T) {
+	t.Run("invalid", func(t *testing.T) {
+		t.Run("source url", func(t *testing.T) {
+			assert.NoError(t, Sync("wtf"))
+			assert.NotZero(t, len(Providers))
+		})
+		t.Run("resource body", func(t *testing.T) {
+			assert.Error(t, Sync("https://ddg.gg/"))
+			assert.NotZero(t, len(Providers))
+		})
+	})
+	t.Run("valid url", func(t *testing.T) {
+		assert.NoError(t, Sync(SourceURL))
+		assert.NotZero(t, len(Providers))
+	})
+}

sync_test.go

@@ -2,12 +2,14 @@
 package oembed
 
 type (
+	// Provider represent a single provider info
 	Provider struct {
 		Name      string     `json:"provider_name"`
 		URL       string     `json:"provider_url"`
 		Endpoints []Endpoint `json:"endpoints"`
 	}
 
+	// Provider represent a single endpoint of Provider
 	Endpoint struct {
 		Schemes   []string `json:"schemes,omitempty"`
 		URL       string   `json:"url"`
@@ -17,7 +19,7 @@ type (
 
 	// Response can specify a resource type, such as photo or video.
 	// Each type has specific parameters associated with it.
-	Response struct {
+	OEmbed struct {
 		// The resource type.
 		Type string `json:"type"` // required
 
@@ -88,7 +90,7 @@ type (
 	// Link type allow a provider to return any generic embed data (such as title and author_name), without
 	// providing either the url or html parameters. The consumer may then link to the resource, using the URL
 	// specified in the original request.
-	Link string
+	// Link string
 
 	// Rich is used for rich HTML content that does not fall under one of the other categories.
 	Rich struct {