Rework regexes & fixtures into local / upstream

Update all tests / regexes / fixtures from upstream
thinkwelltwd · Jun 29, 2018 · 84d1bd0 · 84d1bd0
1 parent d2bba9f
commit 84d1bd0
Show file tree

Hide file tree

Showing 103 changed files with 16,919 additions and 17,101 deletions.
diff --git a/device_detector/device_detector.py b/device_detector/device_detector.py
@@ -7,15 +7,25 @@
     OS,
 
     # Devices
-    Bot, Device,
+    Bot,
+    Device,
 
     # Clients
-    Browser, FeedReader, Library,
-    MediaPlayer, MobileApp, DesktopApp, PIM,
+    Browser,
+    FeedReader,
+    Library,
+    MediaPlayer,
+    MobileApp,
+    DesktopApp,
+    PIM,
 )
 from . import DDCache
 
-MAC_iOS = {'ATV', 'IOS', 'MAC'}
+MAC_iOS = {
+    'ATV',
+    'IOS',
+    'MAC',
+}
 
 
 class DeviceDetector:

diff --git a/device_detector/parser/client/browser.py b/device_detector/parser/client/browser.py
@@ -240,7 +240,7 @@ class Engine(BaseClientParser):
     AVAILABLE_ENGINES = AVAILABLE_ENGINES
 
     fixture_files = [
-        'client/browser_engine.yml',
+        'upstream/client/browser_engine.yml',
     ]
 
     def parse(self):
@@ -256,7 +256,8 @@ def parse(self):
 class Browser(BaseClientParser):
 
     fixture_files = [
-        'client/browsers.yml',
+        'local/client/browsers.yml',
+        'upstream/client/browsers.yml',
     ]
 
     AVAILABLE_ENGINES = AVAILABLE_ENGINES
@@ -292,7 +293,10 @@ def is_mobile_only(self):
 
 
 __all__ = (
-    'Browser', 'Engine',
-    'AVAILABLE_ENGINES', 'AVAILABLE_BROWSERS',
-    'BROWSER_FAMILIES', 'MOBILE_ONLY_BROWSERS',
+    'Browser',
+    'Engine',
+    'AVAILABLE_ENGINES',
+    'AVAILABLE_BROWSERS',
+    'BROWSER_FAMILIES',
+    'MOBILE_ONLY_BROWSERS',
 )
diff --git a/device_detector/parser/client/desktopapp.py b/device_detector/parser/client/desktopapp.py
@@ -4,9 +4,9 @@
 class DesktopApp(BaseClientParser):
 
     fixture_files = [
-        'client/osutility.yml',
-        'client/antivirus.yml',
-        'client/desktop_apps.yml',
+        'local/client/osutility.yml',
+        'local/client/antivirus.yml',
+        'local/client/desktop_apps.yml',
     ]
 
     def dtype(self):

diff --git a/device_detector/parser/client/feed_reader.py b/device_detector/parser/client/feed_reader.py
@@ -7,7 +7,7 @@ def dtype(self):
         return 'feed reader'
 
     fixture_files = [
-        'client/feed_readers.yml',
+        'upstream/client/feed_readers.yml',
     ]
 
 

diff --git a/device_detector/parser/client/library.py b/device_detector/parser/client/library.py
@@ -4,7 +4,8 @@
 class Library(BaseClientParser):
 
     fixture_files = [
-        'client/libraries.yml',
+        'local/client/libraries.yml',
+        'upstream/client/libraries.yml',
     ]
 
 

diff --git a/device_detector/parser/client/mediaplayer.py b/device_detector/parser/client/mediaplayer.py
@@ -4,7 +4,8 @@
 class MediaPlayer(BaseClientParser):
 
     fixture_files = [
-        'client/mediaplayers.yml',
+        'local/client/mediaplayers.yml',
+        'upstream/client/mediaplayers.yml',
     ]
 
 

diff --git a/device_detector/parser/client/mobileapp.py b/device_detector/parser/client/mobileapp.py
@@ -4,7 +4,8 @@
 class MobileApp(BaseClientParser):
 
     fixture_files = [
-        'client/mobile_apps.yml',
+        'local/client/mobile_apps.yml',
+        'upstream/client/mobile_apps.yml',
     ]
 
     def dtype(self):

diff --git a/device_detector/parser/client/pim.py b/device_detector/parser/client/pim.py
@@ -4,7 +4,8 @@
 class PIM(BaseClientParser):
 
     fixture_files = [
-        'client/pim.yml',
+        'local/client/pim.yml',
+        'upstream/client/pim.yml',
     ]
 
 

diff --git a/device_detector/parser/device/base.py b/device_detector/parser/device/base.py
@@ -369,6 +369,7 @@
     'VV': 'Vivo',
     'V1': 'Voto',
     'VO': 'Voxtel',
+    'VR': 'Vernee',
     'VF': 'Vodafone',
     'VZ': 'Vizio',
     'VW': 'Videoweb',
@@ -427,5 +428,6 @@ class BaseDeviceParser(Parser):
 
 
 __all__ = (
-    'BaseDeviceParser', 'DEVICE_BRANDS',
+    'BaseDeviceParser',
+    'DEVICE_BRANDS',
 )
diff --git a/device_detector/parser/device/bot.py b/device_detector/parser/device/bot.py
@@ -4,7 +4,7 @@
 class Bot(BaseDeviceParser):
 
     fixture_files = [
-        'bots.yml',
+        'upstream/bots.yml',
     ]
 
     def is_bot(self) -> bool:

diff --git a/device_detector/parser/device/device.py b/device_detector/parser/device/device.py
@@ -13,11 +13,11 @@ class Device(BaseDeviceParser):
     # parser classes used in the matomo project, except for TVs
     # which are parsed separately
     fixture_files = [
-        'device/consoles.yml',
-        'device/car_browsers.yml',
-        'device/cameras.yml',
-        'device/portable_media_player.yml',
-        'device/mobiles.yml',
+        'upstream/device/consoles.yml',
+        'upstream/device/car_browsers.yml',
+        'upstream/device/cameras.yml',
+        'upstream/device/portable_media_player.yml',
+        'upstream/device/mobiles.yml',
     ]
 
     # -----------------------------------------------------------------------------
@@ -30,7 +30,7 @@ def tv_regex_list(self) -> list:
         if regexes:
             return regexes
 
-        regexes = self.load_from_yaml('device/televisions.yml')
+        regexes = self.load_from_yaml('upstream/device/televisions.yml')
         if not regexes:
             return []
 

diff --git a/device_detector/parser/device/vendor_fragment.py b/device_detector/parser/device/vendor_fragment.py
@@ -3,7 +3,7 @@
 
 class VendorFragment(BaseDeviceParser):
     fixture_files = [
-        'vendorfragments.yml',
+        'upstream/vendorfragments.yml',
     ]
 
     def yaml_to_list(self, yfile) -> list:

diff --git a/device_detector/parser/extractors.py b/device_detector/parser/extractors.py
@@ -108,6 +108,8 @@ def extract(self) -> str:
 
 
 __all__ = (
-    'DataExtractor', 'NameExtractor',
-    'ModelExtractor', 'VersionExtractor',
+    'DataExtractor',
+    'ModelExtractor',
+    'NameExtractor',
+    'VersionExtractor',
 )
diff --git a/device_detector/parser/operating_system.py b/device_detector/parser/operating_system.py
@@ -140,7 +140,8 @@
 class OS(Parser):
 
     fixture_files = [
-        'oss.yml',
+        'local/oss.yml',
+        'upstream/oss.yml',
     ]
 
     DESKTOP_OS = DESKTOP_OS
@@ -198,6 +199,8 @@ def set_details(self) -> None:
 
 
 __all__ = (
+    'DESKTOP_OS',
     'OS',
-    'DESKTOP_OS', 'OPERATING_SYSTEMS', 'OS_FAMILIES',
+    'OPERATING_SYSTEMS',
+    'OS_FAMILIES',
 )
diff --git a/device_detector/parser/os_fragment.py b/device_detector/parser/os_fragment.py
@@ -7,7 +7,7 @@
 
 class OSFragment(BaseDeviceParser):
     fixture_files = [
-        'osfragments.yml',
+        'local/osfragments.yml',
     ]
 
     def yaml_to_list(self, yfile) -> list:

diff --git a/device_detector/parser/parser.py b/device_detector/parser/parser.py
@@ -11,8 +11,16 @@
 import device_detector
 from functools import lru_cache
 from pathlib import Path
-from ..settings import BOUNDED_REGEX, DDCache, ROOT
-from .extractors import NameExtractor, ModelExtractor, VersionExtractor
+from ..settings import (
+    BOUNDED_REGEX,
+    DDCache,
+    ROOT,
+)
+from .extractors import (
+    NameExtractor,
+    ModelExtractor,
+    VersionExtractor,
+)
 
 @lru_cache(None)
 def compiled_regex(regex):

diff --git a/device_detector/regexes/local/__init__.py b/device_detector/regexes/local/__init__.py
diff --git a/device_detector/regexes/local/client/__init__.py b/device_detector/regexes/local/client/__init__.py
diff --git a/device_detector/regexes/client/antivirus.yml → ...tector/regexes/local/client/antivirus.yml b/device_detector/regexes/client/antivirus.yml → ...tector/regexes/local/client/antivirus.yml
@@ -1,3 +1,10 @@
+###############
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
+
 - regex: clamav/(\d+[\.\d]+)
   name: ClamAV
   version: '$1'

diff --git a/device_detector/regexes/local/client/browsers.yml b/device_detector/regexes/local/client/browsers.yml
@@ -0,0 +1,19 @@
+###############
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
+
+# Avastium Browser - Avast browser based on Chrome
+- regex: 'Avastium Chrome(?:\/(\d+[\.\d]+))'
+  name: 'Avast SafeZone'
+  version: '$1'
+  engine:
+    default: 'WebKit'
+
+- regex: 'avastium[ \/]\(?(\d+[\.\d]+)'
+  name: 'Avast SafeZone'
+  version: '$1'
+  engine:
+    default: 'WebKit'
diff --git a/..._detector/regexes/client/desktop_apps.yml → ...tor/regexes/local/client/desktop_apps.yml b/..._detector/regexes/client/desktop_apps.yml → ...tor/regexes/local/client/desktop_apps.yml
@@ -1,3 +1,9 @@
+###############
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
 # General Desktop Apps
 
 - regex: com.apple.Maps

diff --git a/device_detector/regexes/local/client/libraries.yml b/device_detector/regexes/local/client/libraries.yml
@@ -0,0 +1,22 @@
+###############
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
+
+- regex: 'aws-sdk-([a-z]+)\/(\d+[\.\d]+)'
+  name: 'Amazon Web Services SDK for $1'
+  version: '$2'
+
+- regex: 'FB(Android|iOS)SDK\.(\d+[\.\d]+)'
+  name: 'Facebook SDK for $1'
+  version: '$2'
+
+- regex: 'xboxservicesapi/(\d+[\.\d]+)'
+  name: 'Xbox Services API'
+  version: '$1'
+
+- regex: 'zendesk-sdk/(\d+[\.\d]+)'
+  name: 'Zendesk SDK'
+  version: '$1'
diff --git a/device_detector/regexes/local/client/mediaplayers.yml b/device_detector/regexes/local/client/mediaplayers.yml
@@ -0,0 +1,10 @@
+###############
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
+
+- regex: 'Spotify\/(\d+)'
+  name: 'Spotify'
+  version: '$1'
diff --git a/...e_detector/regexes/client/mobile_apps.yml → ...ctor/regexes/local/client/mobile_apps.yml b/...e_detector/regexes/client/mobile_apps.yml → ...ctor/regexes/local/client/mobile_apps.yml
@@ -1,74 +1,10 @@
 ###############
-# Device Detector - The Universal Device Detection library for parsing User Agents
+# Supplemental regexes for Device Detector - The Universal Device Detection library for parsing User Agents
 #
-# @link http://piwik.org
+# Copyright (c) 2018 Dave Burkholder, Thinkwell Designs
 # @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
 ###############
 
-# AndroidDownloadManager
-- regex: 'AndroidDownloadManager(?:[ /]([\d\.]+))?'
-  name: 'AndroidDownloadManager'
-  version: '$1'
-
-# Apple News
-- regex: '(?:Apple)?News(?:[ /][\d\.]+)? Version(?:[ /]([\d\.]+))?'
-  name: 'Apple News'
-  version: '$1'
-
-- regex: 'bPod'
-  name: 'bPod'
-  version: ''
-
-# Facebook Messenger
-- regex: '(?:MessengerForiOS|MESSENGER).(?:FBAV)(?:[ /]([\d\.]+))?'
-  name: 'Facebook Messenger'
-  version: '$1'
-
-# Facebook
-- regex: '(?:FBAV|com.facebook.katana)(?:[ /]([\d\.]+))?'
-  name: 'Facebook'
-  version: '$1'
-
-# FeedR
-- regex: 'FeedR(?:/([\d\.]+))?'
-  name: 'FeedR'
-  version: '$1'
-
-# Google Play Kiosk
-- regex: 'com.google.android.apps.magazines'
-  name: 'Google Play Newsstand'
-  version: ''
-
-# Google Plus
-- regex: 'com.google.GooglePlus'
-  name: 'Google Plus'
-  version: ''
-
-# WeChat
-- regex: 'MicroMessenger/([^ ]+)'
-  name: 'WeChat'
-  version: '$1'
-
-# Sina Weibo
-- regex: '.*__weibo__([0-9\.]+)__'
-  name: 'Sina Weibo'
-  version: '$1'
-
-# Pinterest
-- regex: 'Pinterest(?:/([\d\.]+))?'
-  name: 'Pinterest'
-  version: '$1'
-
-# Podcatcher Deluxe
-- regex: 'Podcatcher Deluxe'
-  name: 'Podcatcher Deluxe'
-  version: ''
-
-# YouTube
-- regex: 'com.google.android.youtube(?:/([\d\.]+))?'
-  name: 'YouTube'
-  version: '$1'
-
 - regex: 'youtube\/([\d\.]+)'
   name: 'YouTube'
   version: '$1'