Exit early if no twitter videos detected (#115)

* Exit early if no twitter videos detected This change make possible extract gif from tweets as mp4 video * Fix test
microlinkhq · Sep 5, 2018 · a69979f · a69979f
1 parent 25cec6f
commit a69979f
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 84 deletions.
diff --git a/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot b/packages/metascraper-media-provider/__snapshots__/index.js.snap-shot
@@ -2,9 +2,3 @@ exports['twitter 1'] = "https://video.twimg.com/amplify_video/943561675927519232
 
 exports['vimeo 1'] = "https://gcs-vimeo.akamaized.net/exp=1535800514~acl=%2A%2F823603782.mp4%2A~hmac=1ff98e8349d222bbb11b2e7e3105729a578f83c632a34e252db61034b2b05d6b/vimeo-prod-skyfire-std-us/01/2635/7/188175573/823603782.mp4"
 
-exports['twitter 2'] = false
-
-exports['vimeo 2'] = false
-
-exports['youtube 1'] = "https://r3---sn-aigl6nek.googlevideo.com/videoplayback?keepalive=yes&fvip=3&mime=audio%2Fmp4&gir=yes&ipbits=0&lmt=1507952811949144&expire=1535818875&pl=24&mm=31%2C26&ip=95.131.170.224&mn=sn-aigl6nek%2Csn-4g5ednss&initcwndbps=301250&ms=au%2Conr&itag=140&mt=1535797130&dur=141.502&mv=m&id=o-AMQjKkqESQJsZjtvcGYi-wuToRTYpm-mUp5Ck3IHofXM&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&ei=G2iKW_jiIMTjV6OjsqgF&key=yt6&requiressl=yes&c=WEB&source=youtube&clen=2248120&signature=E3942E3A3682B75BD7ADA0D3B23D839993029231.CA74D985A1158F2EAD22633C998E28A30F5F28A2&ratebypass=yes"
-
diff --git a/packages/metascraper-media-provider/src/get-media/index.js b/packages/metascraper-media-provider/src/get-media/index.js
@@ -1,11 +1,11 @@
 'use strict'
 
+const { protocol } = require('@metascraper/helpers')
+const { isEmpty, reduce } = require('lodash')
+
 const { isTwitterUrl, getTwitterInfo } = require('./twitter-info')
 const createGetMedia = require('./get-media')
 
-const { protocol } = require('@metascraper/helpers')
-const { chain } = require('lodash')
-
 // Local cache for successive calls
 let cachedVideoInfoUrl
 let cachedVideoInfo
@@ -21,8 +21,11 @@ module.exports = opts => {
       getTwitterInfo(url)
     ])
 
-    const formats = chain(videoInfo.formats)
-      .reduce((acc, format, index) => {
+    if (isEmpty(twitterVideos)) return videoInfo
+
+    const formats = reduce(
+      videoInfo.formats,
+      (acc, format, index) => {
         const { url } = twitterVideos[index]
         const item = {
           ...format,
@@ -31,8 +34,9 @@ module.exports = opts => {
           extractor_key: 'Twitter'
         }
         return [...acc, item]
-      }, [])
-      .value()
+      },
+      []
+    )
 
     return { ...videoInfo, formats }
   }

diff --git a/packages/metascraper-media-provider/src/get-media/twitter-info.js b/packages/metascraper-media-provider/src/get-media/twitter-info.js
@@ -42,6 +42,7 @@ const getTwitterInfo = async url => {
     .get(
       `globalObjects.tweets.${tweetId}.extended_entities.media.0.video_info.variants`
     )
+    .filter('bitrate')
     .orderBy('bitrate', 'asc')
     .value()
 }

diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js
@@ -69,10 +69,11 @@ describe('metascraper-media-provider', () => {
       })
     })
 
-    describe.only('twitter', () => {
+    describe('twitter', () => {
       ;[
         'https://twitter.com/verge/status/957383241714970624',
-        'https://twitter.com/telediario_tve/status/1036860275859775488'
+        'https://twitter.com/telediario_tve/status/1036860275859775488',
+        'https://twitter.com/Mei_Gui8/status/1037374230785142785'
       ].forEach(url => {
         it(url, async () => {
           const metadata = await metascraper({ html: '<title></title>', url })

diff --git a/packages/metascraper-video/__snapshots__/index.js.snap-shot b/packages/metascraper-video/__snapshots__/index.js.snap-shot
@@ -1,77 +1,30 @@
 exports['<source src /> 1'] = {
   "image": "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460s.jpg",
-  "video": "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460sv.mp4",
-  "author": null,
-  "date": null,
-  "description": "Watch the video and the fun convo of the 9GAG community",
-  "lang": "en",
-  "logo": "https://assets-9gag-fun.9cache.com/s/fab0aa49/174d0b1ec891e220e79a7cb3af66a1fbb1eed154/static/dist/core/img/favicon.ico",
-  "publisher": "9GAG",
-  "title": "Newspaperboy in Germany",
-  "url": "http://9gag.com/gag/aGjVLDK"
+  "video": "https://img-9gag-fun.9cache.com/photo/aGjVLDK_460sv.mp4"
 }
 
 exports['og:video 1'] = {
-  "image": "https://pbs.twimg.com/tweet_video_thumb/DUQODu8VMAASlkj.jpg",
-  "video": null,
-  "author": "_developit",
-  "date": "2018-02-18T12:00:00.000Z",
-  "description": "“📣 Announcing... Karmatic!\n🦑 Easy automatic (headless) browser testing\n🛠 No config needed, auto-detects webpack settings\n🤹‍♀️ Only one dependency to juggle\n🏎 Powered by Karma, Webpack, Jasmine &amp; Puppeteer\nhttps://t.co/nguAcGaWQP https://t.co/OgxikFBdQ8”",
-  "lang": "es",
-  "logo": "https://abs.twimg.com/icons/apple-touch-icon-192x192.png",
-  "publisher": "Twitter",
-  "title": "Jason Miller 🦊⚛ on Twitter",
-  "url": "https://twitter.com/_developit/status/955905369242513414"
+  "image": null,
+  "video": null
 }
 
 exports['single src 1'] = {
-  "image": "https://cdn.vox-cdn.com/thumbor/AtQQMyWrexi6-Xyk73jv6nqTO7s=/0x5:1247x658/fit-in/1200x630/cdn.vox-cdn.com/uploads/chorus_asset/file/10079811/Screen_Shot_2018_01_22_at_3.27.50_PM.png",
-  "video": "https://cdn.vox-cdn.com/thumbor/4l0C-7uGFtTfc6lWibo1ITiE2YU=/0x0:1280x720/320x213/filters:focal(538x258:742x462):gifv():no_upscale()/cdn.vox-cdn.com/uploads/chorus_image/image/58416873/2018_01_22_14_19_55.0.gif",
-  "author": "Rachel Becker",
-  "date": "2018-01-22T23:38:17.000Z",
-  "description": "The zombies are only released on the weekends, the developers promise",
-  "lang": null,
-  "logo": "https://cdn.vox-cdn.com/uploads/chorus_asset/file/7395351/android-chrome-192x192.0.png",
-  "publisher": "The Verge",
-  "title": "You can visit the Pentagon’s secret nuclear bunker inside Minecraft",
-  "url": "https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model"
+  "image": null,
+  "video": "https://cdn.vox-cdn.com/thumbor/4l0C-7uGFtTfc6lWibo1ITiE2YU=/0x0:1280x720/320x213/filters:focal(538x258:742x462):gifv():no_upscale()/cdn.vox-cdn.com/uploads/chorus_image/image/58416873/2018_01_22_14_19_55.0.gif"
 }
 
 exports['clips.twitch.tv 1'] = {
   "image": "https://clips-media-assets.twitch.tv/27434665136-offset-2366-preview.jpg",
-  "video": "https://clips-media-assets.twitch.tv/AT-27434665136-offset-2366-1280x720.mp4",
-  "author": null,
-  "date": null,
-  "description": "Shroud with the casual coffee sip to kill combo - Clipped by jpan11",
-  "lang": "en",
-  "logo": "https://www.twitch.tv/favicon.ico",
-  "publisher": "Twitch",
-  "title": "shroud Playing PLAYERUNKNOWN’S BATTLEGROUNDS - Twitch Clips",
-  "url": "https://clips.twitch.tv/AwkwardBoredWaffleItsBoshyTime"
+  "video": "https://clips-media-assets.twitch.tv/AT-27434665136-offset-2366-1280x720.mp4"
 }
 
 exports['play.tv 1'] = {
   "image": "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/720.jpg",
-  "video": "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/480.mp4",
-  "author": "chineseouchie",
-  "description": "Publicado por chineseouchie",
-  "lang": "en",
-  "logo": "https://plays.tv/images/chrome/touch_icon_152x152.png",
-  "publisher": "Plays.tv",
-  "title": "Holy Shit",
-  "url": "https://plays.tv/video/5a6f64b1bef69a7fa9/holy-shit"
+  "video": "https://d1playscdntv-a.akamaihd.net/video/Ha35bprkDYG/processed/480.mp4"
 }
 
 exports['src:poster 1'] = {
   "image": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.jpg",
-  "video": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.mp4",
-  "author": "Gfycat",
-  "date": null,
-  "description": "Watch Backflip GIF on Gfycat. Discover more PUBG GIFs on Gfycat",
-  "lang": "en",
-  "logo": "https://gfycat.com/static/apple-touch-icon/apple-touch-icon-180x180.png",
-  "publisher": "Gfycat",
-  "title": "Backflip - Create, Discover and Share Awesome GIFs on Gfycat",
-  "url": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-size_restricted.gif"
+  "video": "https://thumbs.gfycat.com/TimelyHealthyArmadillo-mobile.mp4"
 }
 
diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js
@@ -6,18 +6,7 @@ const { resolve } = require('path')
 const { omit } = require('lodash')
 const fs = require('fs')
 
-const metascraper = require('metascraper')([
-  require('..')(),
-  require('metascraper-author')(),
-  require('metascraper-date')(),
-  require('metascraper-description')(),
-  require('metascraper-image')(),
-  require('metascraper-lang')(),
-  require('metascraper-logo')(),
-  require('metascraper-publisher')(),
-  require('metascraper-title')(),
-  require('metascraper-url')()
-])
+const metascraper = require('metascraper')([require('..')()])
 
 const readFile = promisify(fs.readFile)
 
@@ -45,8 +34,7 @@ describe('metascraper-video', () => {
         const url =
           'https://www.theverge.com/2018/1/22/16921092/pentagon-secret-nuclear-bunker-reconstruction-minecraft-cns-miis-model'
         const metadata = await metascraper({ html, url })
-        console.log(metadata)
-        // snapshot(metadata)
+        snapshot(metadata)
       })
     })