fix: value detection

packages/metascraper-helpers/index.js

@@ -17,7 +17,8 @@ const {
   size,
   toLower,
   trim,
-  invoke
+  invoke,
+  isNil
 } = require('lodash')
 
 const langs = require('iso-639-3').map(({ iso6391 }) => iso6391)
@@ -31,6 +32,7 @@ const _normalizeUrl = require('normalize-url')
 const smartquotes = require('smartquotes')
 const { decodeHTML } = require('entities')
 const mimeTypes = require('mime-types')
+const hasValues = require('has-values')
 const chrono = require('chrono-node')
 const truncate = require('truncate')
 const isIso = require('isostring')
@@ -285,18 +287,28 @@ const createWrap = (fn, opts) => rule => ({ htmlDom, url }) => {
   return fn(value, opts)
 }
 
+const hasValue = value =>
+  isNil(value) || value === false || value === 0 || value === ''
+    ? false
+    : hasValues(value)
+
 module.exports = {
   $filter,
   $jsonld,
   absoluteUrl,
+  audio,
   author,
+  createValidator,
+  createWrap,
   date,
   description,
   extension,
+  hasValue,
+  image,
   isArray,
-  isAuthor,
   isAudioExtension,
   isAudioUrl,
+  isAuthor,
   isImageExtension,
   isImageUrl,
   isMediaExtension,
@@ -308,18 +320,14 @@ module.exports = {
   isVideoUrl,
   jsonld,
   lang,
+  logo,
   normalizeUrl,
   protocol,
   publisher,
   sanetizeUrl,
   title,
   titleize,
   url,
-  image,
-  logo,
-  audio,
-  video,
   validator,
-  createValidator,
-  createWrap
+  video
 }

packages/metascraper-helpers/package.json

@@ -21,6 +21,7 @@
     "condense-whitespace": "~2.0.0",
     "entities": "~1.1.2",
     "file-extension": "~4.0.5",
+    "has-values": "~2.0.1",
     "image-extensions": "~1.1.0",
     "is-relative-url": "~3.0.0",
     "is-uri": "~1.2.0",

packages/metascraper-helpers/test/index.js

@@ -18,7 +18,8 @@ const {
   description,
   url,
   jsonld,
-  titleize
+  titleize,
+  hasValue
 } = require('..')
 
 describe('metascraper-helpers', () => {
@@ -266,3 +267,24 @@ describe('metascraper-helpers', () => {
     })
   })
 })
+
+describe('.hasValue', () => {
+  it('true', () => {
+    should(hasValue(true)).be.true()
+    should(hasValue('foo')).be.true()
+    should(hasValue(123)).be.true()
+    should(hasValue(['foo'])).be.true()
+    should(hasValue({ foo: 'bar' })).be.true()
+  })
+
+  it('false', () => {
+    should(hasValue(false)).be.false()
+    should(hasValue(0)).be.false()
+    should(hasValue('')).be.false()
+    should(hasValue(null)).be.false()
+    should(hasValue(undefined)).be.false()
+    should(hasValue({})).be.false()
+    should(hasValue([])).be.false()
+    should(hasValue([''])).be.false()
+  })
+})

packages/metascraper/package.json

@@ -61,6 +61,7 @@
     "cheerio": "~1.0.0-rc.2",
     "cheerio-advanced-selectors": "~2.0.1",
     "lodash": "~4.17.13",
+    "map-values-deep": "~1.0.1",
     "whoops": "~4.0.2",
     "xss": "~1.0.6"
   },

packages/metascraper/src/get-data.js

@@ -1,13 +1,8 @@
 'use strict'
 
-const {
-  map,
-  fromPairs,
-  isEmpty,
-  isObject,
-  isArray,
-  mapValues
-} = require('lodash')
+const { isString, map, fromPairs } = require('lodash')
+const { hasValue } = require('@metascraper/helpers')
+const mapValuesDeep = require('map-values-deep')
 
 const xss = require('xss')
 
@@ -18,37 +13,29 @@ const getValue = async ({ htmlDom, url, rules, meta }) => {
   let index = 0
   let value
 
-  while (isEmpty(value) && index < lastIndex) {
+  do {
     const rule = rules[index++]
     const test = rule.test || noopTest
     if (test({ htmlDom, url, meta })) {
       value = await rule({ htmlDom, url, meta })
     }
-  }
+  } while (!hasValue(value) && index < lastIndex)
 
   return value
 }
 
-const mapValuesDeep = (object, fn) => {
-  if (isArray(object)) {
-    return map(object, innerObject => mapValuesDeep(innerObject, fn))
-  }
-
-  if (isObject(object)) {
-    return mapValues(object, value => mapValuesDeep(value, fn))
-  }
-
-  return fn(object)
-}
-
 const escapeValue = (value, { escape }) =>
-  !escape ? value : mapValuesDeep(value, xss)
+  !escape
+    ? value
+    : mapValuesDeep(value, value => (isString(value) ? xss(value) : value))
 
 const getData = async ({ rules, htmlDom, url, escape }) => {
   const data = await Promise.all(
     map(rules, async ([propName, innerRules]) => {
       const rawValue = await getValue({ htmlDom, url, rules: innerRules })
-      const value = isEmpty(rawValue) ? null : escapeValue(rawValue, { escape })
+      const value = hasValue(rawValue)
+        ? escapeValue(rawValue, { escape })
+        : null
       return [propName, value]
     })
   )