🚧 Created worked version of the package

Need documenting source code, checks by linters, add CI config

Author: toby3d

fetch_embed.go

@@ -0,0 +1,53 @@
+package oembed
+
+import (
+	"strconv"
+
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+	template "github.com/valyala/fasttemplate"
+)
+
+type Params struct {
+	MaxWidth  int
+	MaxHeight int
+}
+
+func fetchEmbed(url string, provider providerCandidate, params *Params) (*Response, error) {
+	resourceUrl := provider.URL
+	resourceUrl = template.ExecuteString(resourceUrl, "{", "}", map[string]interface{}{"format": "json"})
+
+	link := http.AcquireURI()
+	defer http.ReleaseURI(link)
+	link.Update(resourceUrl)
+	qa := link.QueryArgs()
+	qa.Add("format", "json")
+	qa.Add("url", url)
+
+	if params != nil && params.MaxWidth != 0 {
+		qa.Add("maxwidth", strconv.Itoa(params.MaxWidth))
+	}
+	if params != nil && params.MaxHeight != 0 {
+		qa.Add("maxheight", strconv.Itoa(params.MaxHeight))
+	}
+	link.SetQueryStringBytes(qa.QueryString())
+
+	req := http.AcquireRequest()
+	defer http.ReleaseRequest(req)
+	req.SetRequestURIBytes(link.FullURI())
+
+	resp := http.AcquireResponse()
+	defer http.ReleaseResponse(resp)
+
+	if err := http.Do(req, resp); err != nil {
+		return nil, err
+	}
+
+	var response Response
+	if err := json.Unmarshal(resp.Body(), &response); err != nil {
+		return nil, err
+	}
+	response.ProviderName = provider.ProviderName
+	response.ProviderURL = provider.ProviderURL
+	return &response, nil
+}

fetch_embed_test.go

@@ -0,0 +1,58 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetchEmbed(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := fetchEmbed(
+			"https://www.youtube.com/watch?v=8jPQjjsBbIc",
+			makeCandidate(Provider{
+				Name: "YouTube",
+				URL:  "https://www.youtube.com/",
+				Endpoints: []Endpoint{Endpoint{
+					Schemes: []string{
+						"https://*.youtube.com/watch*",
+						"https://*.youtube.com/v/*\"",
+						"https://youtu.be/*",
+					},
+					URL:       "https://www.youtube.com/oembed",
+					Discovery: true,
+				}},
+			}),
+			&Params{
+				MaxWidth:  250,
+				MaxHeight: 250,
+			},
+		)
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"",
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				c := findProvider(url)
+				if c == nil {
+					c = new(providerCandidate)
+				}
+
+				_, err := fetchEmbed(url, *c, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}

find_provider.go

@@ -0,0 +1,74 @@
+package oembed
+
+import (
+	"regexp"
+	"strings"
+
+	http "github.com/valyala/fasthttp"
+)
+
+type providerCandidate struct {
+	Domain       string
+	ProviderName string
+	ProviderURL  string
+	Schemes      []string
+	URL          string
+}
+
+func getHostname(url string) string {
+	u := http.AcquireURI()
+	defer http.ReleaseURI(u)
+	u.Update(url)
+	if u.Host() != nil {
+		return strings.TrimPrefix(string(u.Host()), "www.")
+	}
+	return ""
+}
+
+func makeCandidate(p Provider) providerCandidate {
+	endpoint := p.Endpoints[0]
+	domain := getHostname(endpoint.URL)
+	if domain != "" {
+		domain = strings.TrimPrefix(domain, "www.")
+	} else {
+		domain = ""
+	}
+
+	return providerCandidate{
+		ProviderName: p.Name,
+		ProviderURL:  p.URL,
+		Schemes:      endpoint.Schemes,
+		URL:          endpoint.URL,
+		Domain:       domain,
+	}
+
+}
+
+func findProvider(url string) *providerCandidate {
+	var candidates []providerCandidate
+	for _, provider := range providersList {
+		provider := provider
+		candidate := makeCandidate(provider)
+		if len(candidate.Schemes) == 0 {
+			if !strings.Contains(url, candidate.Domain) {
+				continue
+			}
+			candidates = append(candidates, candidate)
+			continue
+		}
+		for _, scheme := range candidate.Schemes {
+			scheme := scheme
+			reg := regexp.MustCompile(strings.Replace(scheme, "*", "(.*)", -1))
+			if !reg.MatchString(url) {
+				continue
+			}
+
+			candidates = append(candidates, candidate)
+			break
+		}
+	}
+	if len(candidates) == 0 {
+		return nil
+	}
+	return &candidates[0]
+}

find_provider_test.go

@@ -0,0 +1,44 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetHostname(t *testing.T) {
+	assert := assert.New(t)
+	for k, v := range map[string]string{
+		"https://mais.uol.com.br/": "mais.uol.com.br",
+		"http://www.wootled.com/":  "wootled.com",
+		"http://yfrog.com":         "yfrog.com",
+		"https://www.youtube.com":  "youtube.com",
+		"https://www.znipe.tv":     "znipe.tv",
+		"http://":                  "",
+		"":                         "",
+	} {
+		t.Run(k, func(t *testing.T) { assert.Equal(v, getHostname(k)) })
+	}
+}
+
+func TestMakeCandidate(t *testing.T) {
+	assert.NotNil(t, makeCandidate(Provider{
+		Name: "YouTube",
+		URL:  "https://www.youtube.com/",
+		Endpoints: []Endpoint{
+			Endpoint{
+				Schemes: []string{
+					"https://*.youtube.com/watch*",
+					"https://*.youtube.com/v/*\"",
+					"https://youtu.be/*",
+				},
+				URL:       "https://www.youtube.com/oembed",
+				Discovery: true,
+			},
+		},
+	}))
+}
+
+func TestFindProvider(t *testing.T) {
+	assert.NotNil(t, findProvider("https://www.beautiful.ai/"))
+}

is_valid_url.go

@@ -0,0 +1,8 @@
+package oembed
+
+import "net/url"
+
+func isValidURL(src string) bool {
+	_, err := url.ParseRequestURI(src)
+	return err == nil
+}

is_valid_url_test.go

@@ -0,0 +1,17 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_IsValidURL(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("invalid", func(t *testing.T) {
+		assert.False(isValidURL("str"))
+	})
+	t.Run("valid", func(t *testing.T) {
+		assert.True(isValidURL("http://www.kickstarter.com"))
+	})
+}

oembed.go

@@ -0,0 +1,24 @@
+package oembed
+
+import (
+	"golang.org/x/xerrors"
+)
+
+var (
+	ErrInvalidInputURL = xerrors.New("invalid input url")
+	ErrNoProviderFound = xerrors.New("no provider found with given url")
+)
+
+func Extract(url string, params *Params) (*Response, error) {
+	if !isValidURL(url) {
+		return nil, ErrInvalidInputURL
+	}
+	if c := findProvider(url); c != nil {
+		return fetchEmbed(url, *c, params)
+	}
+	return nil, ErrNoProviderFound
+}
+
+func HasProvider(url string) bool {
+	return findProvider(url) != nil
+}

oembed_test.go

@@ -0,0 +1,46 @@
+package oembed
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtract(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("valid", func(t *testing.T) {
+		resp, err := Extract("https://www.youtube.com/watch?v=8jPQjjsBbIc", &Params{
+			MaxWidth:  250,
+			MaxHeight: 250,
+		})
+		assert.NoError(err)
+		assert.NotNil(resp)
+	})
+	t.Run("invalid", func(t *testing.T) {
+		for _, url := range []string{
+			"",
+			"htt:/abc.com/failed-none-sense",
+			"https://abc.com/failed-none-sense",
+			"http://badcom/146753785",
+			"https://674458092126388225",
+			"http://www.ted.com/talks/something-does-not-exist",
+			"https://soundcloud^(*%%$%^$$%$$*&(&)())",
+			"https://www.flickr.com/services/oembed/?url=http%3A//www.flickr.com/photos/bees/23416sa/",
+		} {
+			url := url
+			t.Run(url, func(t *testing.T) {
+				_, err := Extract(url, nil)
+				assert.Error(err)
+			})
+		}
+	})
+}
+
+func TestHasProvider(t *testing.T) {
+	t.Run("true", func(t *testing.T) {
+		assert.True(t, HasProvider("https://www.youtube.com/watch?v=8jPQjjsBbIc"))
+	})
+	t.Run("false", func(t *testing.T) {
+		assert.False(t, HasProvider("https://blog.toby3d.me/"))
+	})
+}

sync.go

@@ -0,0 +1,49 @@
+package oembed
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	json "github.com/pquerna/ffjson/ffjson"
+	http "github.com/valyala/fasthttp"
+)
+
+const source string = "https://oembed.com/providers.json"
+
+var (
+	providersList []Provider
+
+	target = filepath.Join(
+		os.Getenv("GOPATH"), "src", "gitlab.com", "toby3d", "oembed", "assets", "providers.json",
+	)
+)
+
+func init() {
+	if err := fetch(source, target); err != nil {
+		panic(err)
+	}
+}
+
+func fetch(url, target string) error {
+	status, src, err := http.Get(nil, url)
+	if err != nil || status != http.StatusOK {
+		if src, err = ioutil.ReadFile(target); err != nil {
+			return err
+		}
+	}
+
+	if err := json.Unmarshal(src, &providersList); err != nil {
+		return err
+	}
+
+	if status == http.StatusOK {
+		if err = ioutil.WriteFile(target, src, os.ModePerm); err != nil {
+			return err
+		}
+	}
+
+	log.Println("providers.json has been updated")
+	return nil
+}