Detect overlapping detections (#7669)

* Detect overlapping detections * fix: added excludes browsers for electron regex fix: added excludes apps and browsers for bot generic regex feat(test): improved error output for DeviceDetectorTest::testParseClient method * fix: added exclude Browser-Maulkorb prefix for Browser-Anonymizer app fix: added exclude SlimerJS prefix for Firefox browser --------- Co-authored-by: Alexsandr Tutik <[email protected]>
matomo-org · Apr 29, 2024 · de9dce9 · de9dce9
1 parent 191fa09
commit de9dce9
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 4 deletions.
diff --git a/Tests/DeviceDetectorTest.php b/Tests/DeviceDetectorTest.php
@@ -268,6 +268,95 @@ public function getFixtures(): array
         return $fixtures;
     }
 
+    /**
+     * @dataProvider getFixturesClient
+     */
+    public function testParseClient(array $fixtureData): void
+    {
+        $ua          = $fixtureData['user_agent'];
+        $clientHints = !empty($fixtureData['headers']) ? ClientHints::factory($fixtureData['headers']) : null;
+
+        AbstractDeviceParser::setVersionTruncation(AbstractDeviceParser::VERSION_TRUNCATION_NONE);
+
+        try {
+            $uaInfo = DeviceDetector::getInfoFromUserAgent($ua, $clientHints);
+        } catch (\Exception $exception) {
+            throw new \Exception(
+                \sprintf('Error: %s from useragent %s', $exception->getMessage(), $ua),
+                $exception->getCode(),
+                $exception
+            );
+        }
+
+        $messageError = \sprintf(
+            "Fixture:\n%s\nResult:\n%s",
+            \Spyc::YAMLDump($fixtureData, 2, 0),
+            \Spyc::YAMLDump($uaInfo, 2, 0)
+        );
+
+        unset($fixtureData['headers']); // ignore headers in result
+        unset($fixtureData['client']['family']);
+
+        $this->assertArrayNotHasKey('bot', $uaInfo, $messageError);
+        $this->assertEquals($fixtureData['client'], $uaInfo['client'], $messageError);
+    }
+
+    public function getFixturesClient(): array
+    {
+        $fixtures     = [];
+        $fixtureFiles = \glob(\realpath(__DIR__) . '/Parser/Client/fixtures/*.yml');
+
+        foreach ($fixtureFiles as $fixturesPath) {
+            $typeFixtures = \Spyc::YAMLLoad($fixturesPath);
+
+            $fixtures = \array_merge(\array_map(static function ($elem) {
+                return [$elem];
+            }, $typeFixtures), $fixtures);
+        }
+
+        return $fixtures;
+    }
+
+    /**
+     * @dataProvider getFixturesDevice
+     */
+    public function testParseDevice(array $fixtureData): void
+    {
+        $ua          = $fixtureData['user_agent'];
+        $clientHints = !empty($fixtureData['headers']) ? ClientHints::factory($fixtureData['headers']) : null;
+
+        AbstractDeviceParser::setVersionTruncation(AbstractDeviceParser::VERSION_TRUNCATION_NONE);
+
+        try {
+            $uaInfo = DeviceDetector::getInfoFromUserAgent($ua, $clientHints);
+        } catch (\Exception $exception) {
+            throw new \Exception(
+                \sprintf('Error: %s from useragent %s', $exception->getMessage(), $ua),
+                $exception->getCode(),
+                $exception
+            );
+        }
+
+        $this->assertArrayNotHasKey('bot', $uaInfo);
+        $this->assertEquals($fixtureData['device'], $uaInfo['device']);
+    }
+
+    public function getFixturesDevice(): array
+    {
+        $fixtures     = [];
+        $fixtureFiles = \glob(\realpath(__DIR__) . '/Parser/Device/fixtures/*.yml');
+
+        foreach ($fixtureFiles as $fixturesPath) {
+            $typeFixtures = \Spyc::YAMLLoad($fixturesPath);
+
+            $fixtures = \array_merge(\array_map(static function ($elem) {
+                return [$elem];
+            }, $typeFixtures), $fixtures);
+        }
+
+        return $fixtures;
+    }
+
     public function testInstanceReusage(): void
     {
         $userAgents = [

diff --git a/regexes/bots.yml b/regexes/bots.yml
@@ -4432,5 +4432,5 @@
   name: 'Generic Bot'
 
 # Generic detections
-- regex: '[a-z0-9_-]*(?:(?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|fetch(?:er)?|indexer|inspector|monitor|project(?!or)|(?<!Google Wap )proxy|research|resolver|robots|scanner|scraper|script|searcher|(?<!-)security|spider|study|transcoder|uptime|user[ _]?agent|validator)(?:[^a-z]|$)'
+- regex: '[a-z0-9_-]*(?:(?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\d\.[x\d] |electron-)fetch(?:er)?|indexer|inspector|monitor|(?<!Microsoft |banshee-)project(?!or)|(?<!Google Wap |Blue )proxy|research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider|study|transcoder|uptime|user[ _]?agent|validator)(?:[^a-z]|$)'
   name: 'Generic Bot'
diff --git a/regexes/client/browsers.yml b/regexes/client/browsers.yml
@@ -3198,7 +3198,7 @@
     default: 'WebKit'
 
 # Firefox
-- regex: '(?<!like )Gecko'
+- regex: '(?<!like )Gecko(?!/\d+ SlimerJS)'
   name: 'Firefox'
   version: ''
   engine:

diff --git a/regexes/client/mobile_apps.yml b/regexes/client/mobile_apps.yml
@@ -1427,7 +1427,7 @@
   version: ''
 
 # Browser-Anonymizer (https://www.almisoft.de/)
-- regex: 'Anonymisiert durch AlMiSoft'
+- regex: 'Anonymisiert durch AlMiSoft(?! Browser-Maulkorb)'
   name: 'Browser-Anonymizer'
   version: ''
 
@@ -2478,7 +2478,7 @@
   version: '$1'
 
 # Electron generic apps
-- regex:  ' (?!(?:AppleWebKit|brave))([a-z0-9]*)(?:-desktop|-electron-app)?/(\d+\.[\d.]+).*Electron/'
+- regex:  ' (?!(?:AppleWebKit|brave|Franz|Mailspring|Notion|Basecamp|Evernote|catalyst|ramboxpro|BlueMail|BeakerBrowser|TweakStyle|Colibri|Polypane|VibeMate|(?:d|LT|Glass|Sushi|Flash|OhHai)Browser|Sizzy))([a-z0-9]*)(?:-desktop|-electron-app)?/(\d+\.[\d.]+).*Electron/'
   name: '$1'
   version: '$2'