fix: avoid use generic publisher (#141)

* fix: avoid use generic publisher * fix: move rules into favicon package * fix: add keywords meta * test: update snapshots * test: add missing dependency * test: update snapshots
microlinkhq · Dec 16, 2018 · c13f833 · c13f833
1 parent fe0c9a2
commit c13f833
Show file tree

Hide file tree

Showing 98 changed files with 225 additions and 84 deletions.
diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "author",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "author",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3",
     "lodash": "~4.17.10"

diff --git a/packages/metascraper-clearbit-logo/package.json b/packages/metascraper-clearbit-logo/package.json
@@ -18,6 +18,7 @@
   },
   "keywords": [
     "clearbit",
+    "logo",
     "metascraper"
   ],
   "dependencies": {

diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "date",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "description",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "helpers",
+    "metascraper"
+  ],
   "dependencies": {
     "audio-extensions": "0.0.0",
     "chrono-node": "~1.3.5",
@@ -44,10 +48,10 @@
   "scripts": {
     "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc mocha test"
   },
+  "license": "MIT",
   "standard": {
     "env": [
       "mocha"
     ]
-  },
-  "license": "MIT"
+  }
 }
diff --git a/packages/metascraper-image/package.json b/packages/metascraper-image/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "image",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-lang-detector/package.json b/packages/metascraper-lang-detector/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "lang",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3",
     "franc": "~4.0.0",

diff --git a/packages/metascraper-lang/__snapshots__/index.js.snap-shot b/packages/metascraper-lang/__snapshots__/index.js.snap-shot
@@ -1,12 +1,4 @@
 exports['html lang property 1'] = {
-  "lang": "pl",
-  "author": "Jakub Majmurek",
-  "title": "Churchill, bohater naszych czasów / Film / dwutygodnik.com",
-  "publisher": "Dwutygodnik",
-  "image": "http://www.dwutygodnik.com/public/media/article/image_full/7615.png",
-  "date": "2018-01-01T12:00:00.000Z",
-  "description": "Wysyp filmów o Churchillu w pobrexitowej Brytanii, wydaje się czymś zrozumiałym. Uosabia on ostatni moment prawdziwej wielkości Zjednoczonego Królestwa – wspomnienie tej historycznej chwili pozwala oswoić traumy i lęki",
-  "logo": "http://www.dwutygodnik.com/public/frontend/image_v4/favicon.ico",
-  "url": "http://www.dwutygodnik.com/artykul/7615-churchill-bohater-naszych-czasow.html"
+  "lang": "pl"
 }
 
diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "lang",
+    "metascraper"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-lang/test/index.js b/packages/metascraper-lang/test/index.js
@@ -5,18 +5,7 @@ const { promisify } = require('util')
 const { resolve } = require('path')
 const fs = require('fs')
 
-const metascraper = require('metascraper')([
-  require('metascraper-amazon')(),
-  require('metascraper-author')(),
-  require('metascraper-date')(),
-  require('metascraper-description')(),
-  require('metascraper-image')(),
-  require('..')(),
-  require('metascraper-logo')(),
-  require('metascraper-publisher')(),
-  require('metascraper-title')(),
-  require('metascraper-url')()
-])
+const metascraper = require('metascraper')([require('..')()])
 
 const readFile = promisify(fs.readFile)
 

diff --git a/packages/metascraper-logo-favicon/index.js b/packages/metascraper-logo-favicon/index.js
@@ -1,12 +1,73 @@
 'use strict'
 
+const { flow, first, toNumber, split, chain, concat } = require('lodash')
 const { resolve: resolveUrl, URL } = require('url')
+const { url: urlFn } = require('@metascraper/helpers')
+const got = require('got')
 
-const getFaviconUrl = url => {
-  const {origin} = new URL(url)
-  return resolveUrl(origin, 'favicon.ico')
+const getSize = flow([str => split(str, 'x'), first, toNumber])
+
+const getDomNodeSizes = (domNodes, attr) =>
+  chain(domNodes)
+    .map(({ attribs }) => ({
+      size: getSize(attribs.sizes),
+      link: attribs[attr]
+    }))
+    .value()
+
+const getSizes = ($, collection) =>
+  chain(collection)
+    .reduce((acc, { tag, attr }) => {
+      const domNodes = $(tag).get()
+      const selectors = getDomNodeSizes(domNodes, attr)
+      return concat(acc, selectors)
+    }, [])
+    .sortBy(({ size }) => -size)
+    .value()
+
+const sizeSelectors = [
+  { tag: 'link[rel="apple-touch-icon"]', attr: 'href' },
+  { tag: 'link[rel="apple-touch-icon-precomposed"]', attr: 'href' },
+  { tag: 'meta[name="msapplication-TileImage"]', attr: 'content' },
+  { tag: 'link[rel="icon"]', attr: 'href' },
+  { tag: 'link[rel="shortcut icon"]', attr: 'href' }
+]
+
+/**
+ * Wrap a rule with validation and formatting logic.
+ *
+ * @param {Function} rule
+ * @return {Function} wrapped
+ */
+
+const wrap = rule => ({ htmlDom, url }) => {
+  const value = rule(htmlDom)
+  return urlFn(value, { url })
 }
 
+/**
+ * Rules.
+ */
 module.exports = () => ({
-  logo: [({ htmlDom: $, meta, url }) => getFaviconUrl(url)]
+  logo: [
+    wrap($ => {
+      const sizes = getSizes($, sizeSelectors)
+      const size = chain(sizes)
+        .first()
+        .get('link')
+        .value()
+      return size
+    }),
+    async ({ url }) => {
+      const { origin } = new URL(url)
+      const logoUrl = resolveUrl(origin, 'favicon.ico')
+
+      try {
+        await got.head(logoUrl)
+        return logoUrl
+      } catch (err) {
+        return null
+      }
+    }
+  ]
 })
diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json
@@ -17,11 +17,16 @@
     "url": "https://github.com/microlinkhq/metascraper-logo-favicon/issues"
   },
   "keywords": [
-    "clearbit",
+    "favicon",
+    "logo",
     "metascraper"
   ],
+  "dependencies": {
+    "@metascraper/helpers": "~4.8.3",
+    "got": "~9.4.0",
+    "lodash": "~4.17.11"
+  },
   "devDependencies": {
-    "lodash": "latest",
     "mocha": "latest",
     "nyc": "latest",
     "should": "latest",

diff --git a/packages/metascraper-logo/index.js b/packages/metascraper-logo/index.js
@@ -1,36 +1,7 @@
 'use strict'
 
-const { flow, chain, first, concat, toNumber, split } = require('lodash')
 const { url: urlFn } = require('@metascraper/helpers')
 
-const getSize = flow([str => split(str, 'x'), first, toNumber])
-
-const getDomNodeSizes = (domNodes, attr) =>
-  chain(domNodes)
-    .map(({ attribs }) => ({
-      size: getSize(attribs.sizes),
-      link: attribs[attr]
-    }))
-    .value()
-
-const getSizes = ($, collection) =>
-  chain(collection)
-    .reduce((acc, { tag, attr }) => {
-      const domNodes = $(tag).get()
-      const selectors = getDomNodeSizes(domNodes, attr)
-      return concat(acc, selectors)
-    }, [])
-    .sortBy(({ size }) => -size)
-    .value()
-
-const sizeSelectors = [
-  { tag: 'link[rel="apple-touch-icon"]', attr: 'href' },
-  { tag: 'link[rel="apple-touch-icon-precomposed"]', attr: 'href' },
-  { tag: 'meta[name="msapplication-TileImage"]', attr: 'content' },
-  { tag: 'link[rel="icon"]', attr: 'href' },
-  { tag: 'link[rel="shortcut icon"]', attr: 'href' }
-]
-
 /**
  * Wrap a rule with validation and formatting logic.
  *
@@ -46,19 +17,10 @@ const wrap = rule => ({ htmlDom, url }) => {
 /**
  * Rules.
  */
-
 module.exports = () => ({
   logo: [
     wrap($ => $('meta[property="og:logo"]').attr('content')),
     wrap($ => $('meta[itemprop="logo"]').attr('content')),
-    wrap($ => $('img[itemprop="logo"]').attr('src')),
-    wrap($ => {
-      const sizes = getSizes($, sizeSelectors)
-      const size = chain(sizes)
-        .first()
-        .get('link')
-        .value()
-      return size
-    })
+    wrap($ => $('img[itemprop="logo"]').attr('src'))
   ]
 })
diff --git a/packages/metascraper-logo/package.json b/packages/metascraper-logo/package.json
@@ -11,10 +11,12 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "logo",
+    "metascraper"
+  ],
   "dependencies": {
-    "@metascraper/helpers": "^4.8.3",
-    "got": "~9.4.0",
-    "lodash": "~4.17.10"
+    "@metascraper/helpers": "^4.8.3"
   },
   "devDependencies": {
     "standard": "11"

diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json
@@ -11,6 +11,11 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "media",
+    "metascraper",
+    "youtube-dl"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3",
     "got": "~9.4.0",

diff --git a/packages/metascraper-media-provider/src/index.js b/packages/metascraper-media-provider/src/index.js
@@ -68,8 +68,8 @@ const getAudio = data => getAudioUrls(data, [hasAudio, isHttps])
 const getAuthor = ({ uploader, creator, uploader_id: uploaderId }) =>
   find([creator, uploader, uploaderId], str => authorFn(str))
 
-const getPublisher = ({ extractor_key: extractorKey }) =>
-  publisher(extractorKey)
+const getPublisher = ({ extractor, extractor_key: extractorKey }) =>
+  extractor !== 'generic' && publisher(extractorKey)
 
 const getLang = ({ language, http_headers: headers = {} }) =>
   lang(language || headers['Accept-Language'])

diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "metascraper",
+    "publisher"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },

diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json
@@ -17,8 +17,9 @@
     "url": "https://github.com/microlinkhq/metascraper-soundcloud/issues"
   },
   "keywords": [
-    "clearbit",
-    "metascraper"
+    "audio",
+    "metascraper",
+    "soundcloud"
   ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"

diff --git a/packages/metascraper-soundcloud/test/index.js b/packages/metascraper-soundcloud/test/index.js
@@ -13,6 +13,7 @@ const metascraper = require('metascraper')([
   require('metascraper-image')(),
   require('metascraper-lang')(),
   require('metascraper-logo')(),
+  require('metascraper-logo-favicon')(),
   require('metascraper-publisher')(),
   require('metascraper-title')(),
   require('metascraper-url')()

diff --git a/packages/metascraper-title/package.json b/packages/metascraper-title/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "metascraper",
+    "title"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3",
     "lodash": "~4.17.10"

diff --git a/packages/metascraper-url/package.json b/packages/metascraper-url/package.json
@@ -11,6 +11,10 @@
   "bugs": {
     "url": "https://github.com/microlinkhq/metascraper/issues"
   },
+  "keywords": [
+    "metascraper",
+    "url"
+  ],
   "dependencies": {
     "@metascraper/helpers": "^4.8.3"
   },