From cddb161c0d72ae9ca003c5125a20b80f16797214 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 22 Feb 2021 10:30:13 +0100 Subject: [PATCH 01/18] extract device type from user agent info --- .../ingest/useragent/DeviceTypeParser.java | 174 ++++++++++++++ .../useragent/IngestUserAgentPlugin.java | 9 +- .../ingest/useragent/UserAgentParser.java | 135 ++++++----- .../ingest/useragent/UserAgentProcessor.java | 2 + .../main/resources/device_type_regexes.yml | 54 +++++ .../useragent/DeviceTypeParserTests.java | 212 ++++++++++++++++++ .../useragent/UserAgentProcessorTests.java | 38 +++- .../src/test/resources/desktop-devices.yml | 174 ++++++++++++++ .../src/test/resources/mobile-devices.yml | 124 ++++++++++ .../src/test/resources/other-devices.yml | 22 ++ .../src/test/resources/robot-devices.yml | 123 ++++++++++ .../src/test/resources/tablet-devices.yml | 29 +++ 12 files changed, 1029 insertions(+), 67 deletions(-) create mode 100644 modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java create mode 100644 modules/ingest-user-agent/src/main/resources/device_type_regexes.yml create mode 100644 modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java create mode 100644 modules/ingest-user-agent/src/test/resources/desktop-devices.yml create mode 100644 modules/ingest-user-agent/src/test/resources/mobile-devices.yml create mode 100644 modules/ingest-user-agent/src/test/resources/other-devices.yml create mode 100644 modules/ingest-user-agent/src/test/resources/robot-devices.yml create mode 100644 modules/ingest-user-agent/src/test/resources/tablet-devices.yml diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java new file mode 100644 index 0000000000000..5ef4a6dc1eb60 --- /dev/null +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.*; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; + +public class DeviceTypeParser { + + private static final String OS_PARSERS = "os_parsers"; + private static final String BROWSER_PARSER = "browser_parsers"; + private static final String DEVICE_PARSER = "device_parsers"; + + private final List patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER); + + private final HashMap> deviceTypePatterns = new HashMap<>(); + + + public void init(InputStream regexStream) throws IOException { + // EMPTY is safe here because we don't use namedObject + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + String currentName = yamlParser.currentName(); + if (token == XContentParser.Token.FIELD_NAME && patternListKeys.contains(currentName)) { + List> parserConfigurations = readParserConfigurations(yamlParser); + ArrayList subPatterns = new ArrayList<>(); + for (Map map : parserConfigurations) { + subPatterns.add(new DeviceTypeSubPattern(Pattern.compile((map.get("regex"))), + map.get("replacement"))); + } + deviceTypePatterns.put(currentName, subPatterns); + } + } + } + + if (patternListKeys.size() != deviceTypePatterns.size()) { + throw new ElasticsearchParseException("not a valid regular expression file"); + } + } + + public String findDeviceType(UserAgentParser.VersionedName userAgent, + UserAgentParser.VersionedName os, UserAgentParser.VersionedName device) { + ArrayList extractedDeviceTypes = new ArrayList<>(); + + String robot = "Robot", tablet = "Tablet", desktop = "Desktop", mobile = "Mobile"; + + if (userAgent == null) { + return null; + } + + for (String patternKey : patternListKeys) { + String deviceType = null; + switch (patternKey) { + case OS_PARSERS: + if (os != null && os.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), os.name); + } + break; + case BROWSER_PARSER: + if (userAgent.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), userAgent.name); + } + break; + case DEVICE_PARSER: + if (device != null && device.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), device.name); + } + break; + default: + break; + } + + if (deviceType != null) { + extractedDeviceTypes.add(deviceType); + } + } + + + if (extractedDeviceTypes.contains(robot)) { + return robot; + } + if (extractedDeviceTypes.contains(tablet)) { + return tablet; + } + if (extractedDeviceTypes.contains(mobile)) { + return mobile; + } + if (extractedDeviceTypes.contains(desktop)) { + return desktop; + } + + return "Other"; + } + + private String findMatch(List possiblePatterns, String matchString) { + String name; + for (DeviceTypeSubPattern pattern : possiblePatterns) { + name = pattern.match(matchString); + if (name != null) { + return name; + } + } + return null; + } + + + static final class DeviceTypeSubPattern { + private final Pattern pattern; + private final String nameReplacement; + + DeviceTypeSubPattern(Pattern pattern, String nameReplacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + } + + public String match(String matchString) { + String name = null; + + Matcher matcher = pattern.matcher(matchString); + + if (matcher.find() == false) { + return null; + } + + int groupCount = matcher.groupCount(); + + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; + } + } + + return name; + } + } + +} diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index e0bd7b7207aae..4a2aae05e45ce 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -39,7 +39,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin { private final Setting CACHE_SIZE_SETTING = Setting.longSetting("ingest.user_agent.cache_size", 1000, 0, - Setting.Property.NodeScope); + Setting.Property.NodeScope); static final String DEFAULT_PARSER_NAME = "_default_"; @@ -66,19 +66,20 @@ static Map createUserAgentParsers(Path userAgentConfigD Map userAgentParsers = new HashMap<>(); UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, - IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), cache); + IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), + IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"), cache); userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yml"); try (Stream regexFiles = Files.find(userAgentConfigDirectory, 1, - (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { + (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { Iterable iterable = regexFiles::iterator; for (Path path : iterable) { String parserName = path.getFileName().toString(); try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream,regexStream, cache)); } } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index a2d388f231dc9..a6cc6e51ac89a 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; + import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -34,20 +35,27 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; + final class UserAgentParser { private final UserAgentCache cache; + + private final DeviceTypeParser deviceTypeParser = new DeviceTypeParser(); + private final List uaPatterns = new ArrayList<>(); private final List osPatterns = new ArrayList<>(); private final List devicePatterns = new ArrayList<>(); + + private final String name; - UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { + UserAgentParser(String name, InputStream regexStream, InputStream deviceTypeRegexStream, UserAgentCache cache) { this.name = name; this.cache = cache; try { init(regexStream); + this.deviceTypeParser.init(deviceTypeRegexStream); } catch (IOException e) { throw new ElasticsearchParseException("error parsing regular expression file", e); } @@ -69,25 +77,23 @@ private void init(InputStream regexStream) throws IOException { for (Map map : parserConfigurations) { uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), - map.get("v3_replacement"), map.get("v4_replacement"))); + map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), + map.get("v3_replacement"), map.get("v4_replacement"))); } - } - else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { + } else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { List> parserConfigurations = readParserConfigurations(yamlParser); for (Map map : parserConfigurations) { osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), - map.get("os_v3_replacement"), map.get("os_v4_replacement"))); + map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), + map.get("os_v3_replacement"), map.get("os_v4_replacement"))); } - } - else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { + } else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { List> parserConfigurations = readParserConfigurations(yamlParser); for (Map map : parserConfigurations) { devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("device_replacement"), null, null, null, null)); + map.get("device_replacement"), null, null, null, null)); } } } @@ -107,8 +113,8 @@ private Pattern compilePattern(String regex, String regex_flag) { } } - private List> readParserConfigurations(XContentParser yamlParser) throws IOException { - List > patternList = new ArrayList<>(); + static public List> readParserConfigurations(XContentParser yamlParser) throws IOException { + List> patternList = new ArrayList<>(); XContentParser.Token token = yamlParser.nextToken(); if (token != XContentParser.Token.START_ARRAY) { @@ -167,8 +173,10 @@ public Details parse(String agentString) { VersionedName userAgent = findMatch(uaPatterns, agentString); VersionedName operatingSystem = findMatch(osPatterns, agentString); VersionedName device = findMatch(devicePatterns, agentString); + String deviceType = deviceTypeParser.findDeviceType(userAgent, operatingSystem, device); + - details = new Details(userAgent, operatingSystem, device); + details = new Details(userAgent, operatingSystem, device, deviceType); cache.put(name, agentString, details); } @@ -193,11 +201,13 @@ static final class Details { public final VersionedName userAgent; public final VersionedName operatingSystem; public final VersionedName device; + public final String deviceType; - Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) { + Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device, String deviceType) { this.userAgent = userAgent; this.operatingSystem = operatingSystem; this.device = device; + this.deviceType = deviceType; } } @@ -208,6 +218,7 @@ static final class VersionedName { public final String patch; public final String build; + VersionedName(String name, String major, String minor, String patch, String build) { this.name = name; this.major = major; @@ -225,60 +236,60 @@ static final class UserAgentSubpattern { private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement; UserAgentSubpattern(Pattern pattern, String nameReplacement, - String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { - this.pattern = pattern; - this.nameReplacement = nameReplacement; - this.v1Replacement = v1Replacement; - this.v2Replacement = v2Replacement; - this.v3Replacement = v3Replacement; - this.v4Replacement = v4Replacement; + String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + this.v1Replacement = v1Replacement; + this.v2Replacement = v2Replacement; + this.v3Replacement = v3Replacement; + this.v4Replacement = v4Replacement; } public VersionedName match(String agentString) { - String name = null, major = null, minor = null, patch = null, build = null; - Matcher matcher = pattern.matcher(agentString); + String name = null, major = null, minor = null, patch = null, build = null; + Matcher matcher = pattern.matcher(agentString); + + if (matcher.find() == false) { + return null; + } + + int groupCount = matcher.groupCount(); - if (matcher.find() == false) { - return null; - } + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; + } + } else if (groupCount >= 1) { + name = matcher.group(1); + } - int groupCount = matcher.groupCount(); + if (v1Replacement != null) { + major = v1Replacement; + } else if (groupCount >= 2) { + major = matcher.group(2); + } - if (nameReplacement != null) { - if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { - name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); - } else { - name = nameReplacement; + if (v2Replacement != null) { + minor = v2Replacement; + } else if (groupCount >= 3) { + minor = matcher.group(3); } - } else if (groupCount >= 1) { - name = matcher.group(1); - } - - if (v1Replacement != null) { - major = v1Replacement; - } else if (groupCount >= 2) { - major = matcher.group(2); - } - - if (v2Replacement != null) { - minor = v2Replacement; - } else if (groupCount >= 3) { - minor = matcher.group(3); - } - - if (v3Replacement != null) { - patch = v3Replacement; - } else if (groupCount >= 4) { - patch = matcher.group(4); - } - - if (v4Replacement != null) { - build = v4Replacement; - } else if (groupCount >= 5) { - build = matcher.group(5); - } - - return name == null ? null : new VersionedName(name, major, minor, patch, build); + + if (v3Replacement != null) { + patch = v3Replacement; + } else if (groupCount >= 4) { + patch = matcher.group(4); + } + + if (v4Replacement != null) { + build = v4Replacement; + } else if (groupCount >= 5) { + build = matcher.group(5); + } + + return name == null ? null : new VersionedName(name, major, minor, patch, build); } - } + } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index f5966f000003b..87e7cd8c2ad0e 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -136,8 +136,10 @@ public IngestDocument execute(IngestDocument ingestDocument) { Map deviceDetails = new HashMap<>(1); if (uaClient.device != null && uaClient.device.name != null) { deviceDetails.put("name", uaClient.device.name); + deviceDetails.put("type", uaClient.deviceType); } else { deviceDetails.put("name", "Other"); + deviceDetails.put("type", "Other"); } uaDetails.put("device", deviceDetails); break; diff --git a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml new file mode 100644 index 0000000000000..a96ef173d2489 --- /dev/null +++ b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml @@ -0,0 +1,54 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Custom parser being added to support device types + +os_parsers: + # Robot + - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' + replacement: 'Robot' + # Desktop OS, Most Common + - regex: '^(Windows$|Windows NT$|Mac OS X|Linux$|Chrome OS|Fedora$|Ubuntu$)' + replacement: 'Desktop' + # Mobile OS + - regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)' + replacement: 'Mobile' + # Desktop OS, Not Common + - regex: '^(FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' + replacement: 'Desktop' + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + replacement: 'Tablet' + +browser_parsers: + # Robot + - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' + replacement: 'Robot' + # Desktop Browsers + - regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)' + replacement: 'Desktop' + # Mobile Browsers, Most Common + - regex: '^(Chrome Mobile|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' + replacement: 'Mobile' + # Mobile Browsers, Not Common + - regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)' + replacement: 'Mobile' + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + replacement: 'Tablet' + +device_parsers: + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + replacement: 'Tablet' diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java new file mode 100644 index 0000000000000..a720f63952dc1 --- /dev/null +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -0,0 +1,212 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; + + +import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; +import static org.hamcrest.Matchers.*; + +public class DeviceTypeParserTests extends ESTestCase { + + private static DeviceTypeParser deviceTypeParser; + + + private ArrayList> readTestDevices(InputStream regexStream, String keyName) throws IOException { + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + ArrayList> testDevices = new ArrayList<>(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + String currentName = yamlParser.currentName(); + if (token == XContentParser.Token.FIELD_NAME && currentName.equals(keyName)) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + HashMap testDevice = new HashMap<>(); + + testDevice.put("type", map.get("type")); + testDevice.put("os", map.get("os")); + testDevice.put("browser", map.get("browser")); + testDevices.add(testDevice); + + } + } + } + } + + return testDevices; + } + + + @BeforeClass + public static void setupDeviceParser() throws IOException { + InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml"); + + assertNotNull(deviceTypeRegexStream); + assertNotNull(deviceTypeRegexStream); + + deviceTypeParser = new DeviceTypeParser(); + deviceTypeParser.init(deviceTypeRegexStream); + } + + @SuppressWarnings("unchecked") + public void testMacDesktop() throws Exception { + VersionedName os = new VersionedName("Mac OS X", null, null, null, null); + + VersionedName userAgent = new VersionedName("Chrome", null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Desktop")); + } + + @SuppressWarnings("unchecked") + public void testAndroidMobile() throws Exception { + + VersionedName os = new VersionedName("iOS", null, null, null, null); + + VersionedName userAgent = new VersionedName("Safari", null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Mobile")); + } + + @SuppressWarnings("unchecked") + public void testIPadTablet() throws Exception { + + VersionedName os = new VersionedName("iOS", null, null, null, null); + + VersionedName userAgent = new VersionedName("Safari", null, null, null, null); + + VersionedName device = new VersionedName("iPad", null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); + + assertThat(deviceType, is("Tablet")); + } + + @SuppressWarnings("unchecked") + public void testWindowDesktop() throws Exception { + + VersionedName os = new VersionedName("Mac OS X", null, null, null, null); + + VersionedName userAgent = new VersionedName("Chrome", null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Desktop")); + } + + @SuppressWarnings("unchecked") + public void testRobotDevices() throws Exception { + + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/robot-devices.yml"); + + ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "robot_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + + VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Robot")); + } + } + + @SuppressWarnings("unchecked") + public void testDesktopDevices() throws Exception { + + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/desktop-devices.yml"); + + ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "desktop_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + + VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + if(deviceType.equals("Mobile")){ + String name = "wow"; + } + assertThat(deviceType, is("Desktop")); + } + } + + @SuppressWarnings("unchecked") + public void testMobileDevices() throws Exception { + + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/mobile-devices.yml"); + + ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "mobile_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + + VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Mobile")); + } + } + + @SuppressWarnings("unchecked") + public void testTabletDevices() throws Exception { + + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/tablet-devices.yml"); + + ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "tablet_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + + VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Tablet")); + } + } + +} diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index e2d2e93706269..f5ca09cad8149 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -43,9 +43,12 @@ public class UserAgentProcessorTests extends ESTestCase { @BeforeClass public static void setupProcessor() throws IOException { InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml"); + InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml"); + assertNotNull(regexStream); + assertNotNull(deviceTypeRegexStream); - UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); + UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream,deviceTypeRegexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser, EnumSet.allOf(UserAgentProcessor.Property.class), false); @@ -112,6 +115,7 @@ public void testCommonBrowser() throws Exception { assertThat(target.get("os"), is(os)); Map device = new HashMap<>(); device.put("name", "Mac"); + device.put("type", "Desktop"); assertThat(target.get("device"), is(device)); } @@ -140,6 +144,7 @@ public void testUncommonDevice() throws Exception { Map device = new HashMap<>(); device.put("name", "Motorola Xoom"); + device.put("type", "Mobile"); assertThat(target.get("device"), is(device)); } @@ -163,6 +168,36 @@ public void testSpider() throws Exception { Map device = new HashMap<>(); device.put("name", "Spider"); + device.put("type", "Robot"); + assertThat(target.get("device"), is(device)); + } + + @SuppressWarnings("unchecked") + public void testTablet() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Mobile Safari")); + + assertThat(target.get("version"), is("12.1")); + + Map os = new HashMap<>(); + os.put("name", "iOS"); + os.put("version", "12.2"); + os.put("full", "iOS 12.2"); + assertThat(target.get("os"), is(os)); + + Map device = new HashMap<>(); + device.put("name", "iPad"); + device.put("type", "Tablet"); assertThat(target.get("device"), is(device)); } @@ -188,6 +223,7 @@ public void testUnknown() throws Exception { assertNull(target.get("os")); Map device = new HashMap<>(); device.put("name", "Other"); + device.put("type", "Other"); assertThat(target.get("device"), is(device)); } } diff --git a/modules/ingest-user-agent/src/test/resources/desktop-devices.yml b/modules/ingest-user-agent/src/test/resources/desktop-devices.yml new file mode 100644 index 0000000000000..90c12d564da95 --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/desktop-devices.yml @@ -0,0 +1,174 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +desktop_devices: + - type: Desktop + os: Windows + browser: Chrome + - type: Desktop + os: Mac OS X + browser: Chrome + - type: Desktop + os: Linux + browser: Cypress + - type: Desktop + os: Ubuntu + browser: Firefox + - type: Desktop + os: Chrome OS + browser: Chrome + - type: Desktop + os: Fedora + browser: Firefox + - type: Desktop + os: FreeBSD + browser: Chrome + - type: Desktop + os: OpenBSD + browser: Firefox + - type: Desktop + os: Arch Linux + browser: Firefox + - type: Desktop + os: Solaris + browser: Firefox + - type: Desktop + os: NetBSD + browser: Firefox + - type: Desktop + os: SUSE + browser: Epiphany + - type: Desktop + browser: Chrome + os: Mac OS X + - type: Desktop + browser: Firefox + os: Windows NT + - type: Desktop + browser: Edge + os: Windows NT + - type: Desktop + browser: HeadlessChrome + os: Linux + - type: Desktop + browser: Safari + os: Mac OS X + - type: Desktop + browser: Electron + os: Linux + - type: Desktop + browser: Opera + os: Linux + - type: Desktop + browser: Samsung Internet + os: Linux + - type: Desktop + browser: Chromium + os: Ubuntu + - type: Desktop + browser: Yandex Browser + os: Windows NT + - type: Desktop + browser: Whale + os: Windows NT + - type: Desktop + browser: Sogou Explorer + os: Windows NT + - type: Desktop + browser: QQ Browser + os: Windows NT + - type: Desktop + browser: IE + os: Windows NT + - type: Desktop + browser: Yeti + os: Windows NT + - type: Desktop + browser: Apple Mail + os: Mac OS X + - type: Desktop + browser: Coc Coc + os: Windows NT + - type: Desktop + browser: Maxthon + os: Windows NT + - type: Desktop + browser: Waterfox + os: Linux + - type: Desktop + browser: Iron + os: Mac OS X + - type: Desktop + browser: UC Browser + os: Windows NT + - type: Desktop + browser: Pale Moon + os: Linux + - type: Desktop + browser: WordPress + os: Linux + - type: Desktop + browser: Vivaldi + os: Windows NT + - type: Desktop + browser: Dragon + os: Windows NT + - type: Desktop + browser: SeaMonkey + os: Windows NT + - type: Desktop + browser: Sleipnir + os: Windows NT + - type: Desktop + browser: Thunderbird + os: Linux + - type: Desktop + browser: Epiphany + os: Linux + - type: Desktop + browser: Datanyze + os: Linux + - type: Desktop + browser: Basilisk + os: Windows NT + - type: Desktop + browser: Swiftfox + os: Linux + - type: Desktop + browser: Netscape + os: SunOS + - type: Desktop + browser: Puffin + os: Linux + - type: Desktop + browser: Seznam prohlížeč + os: Windows NT + - type: Desktop + browser: iCab + os: Mac OS X + - type: Desktop + browser: Opera Neon + os: Windows NT + - type: Desktop + browser: Mail.ru Chromium Browser + os: Windows NT + - type: Desktop + browser: Otter + os: BeOS/Haiku + - type: Desktop + browser: Iceweasel + os: Linux diff --git a/modules/ingest-user-agent/src/test/resources/mobile-devices.yml b/modules/ingest-user-agent/src/test/resources/mobile-devices.yml new file mode 100644 index 0000000000000..340251ba06b16 --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/mobile-devices.yml @@ -0,0 +1,124 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mobile_devices: + - type: Phone + os: Android + browser: Chrome + - type: Phone + os: iOS + browser: Firefox +# - type: Phone +# os: Windows Phone +# browser: Edge + - type: Phone + os: KaiOS + browser: Firefox + - type: Phone + os: Sailfish + browser: SailfishBrowser + - type: Phone + os: Maemo + browser: Fennec + - type: Phone + os: BlackBerry OS + browser: Mobile Safari + - type: Phone + browser: Chrome Mobile + os: Android + - type: Phone + browser: Mobile Safari + os: iOS + - type: Phone + browser: Chrome Mobile WebView + os: Android + - type: Phone + browser: Firefox Mobile + os: Android + - type: Phone + browser: Chrome Mobile iOS + os: iOS + - type: Phone + browser: Facebook + os: Android + - type: Phone + browser: Mobile Safari UI/WKWebView + os: iOS + - type: Phone + browser: Firefox iOS + os: iOS + - type: Phone + browser: Opera Mobile + os: Android + - type: Phone + browser: MiuiBrowser + os: Android + - type: Phone + browser: Edge Mobile + os: Android + - type: Phone + browser: Android + os: Android + - type: Phone + browser: LINE + os: iOS + - type: Phone + browser: QQ Browser Mobile + os: Android + - type: Phone + browser: Flipboard + os: Android + - type: Phone + browser: Instagram + os: iOS + - type: Phone + browser: Pinterest + os: iOS + - type: Phone + browser: OktaMobile + os: iOS + - type: Phone + browser: Twitter + os: Android + - type: Phone + browser: Mint Browser + os: Android + - type: Phone + browser: Snapchat + os: iOS + - type: Phone + browser: IE Mobile + os: Windows Phone + - type: Phone + browser: Sailfish Browser + os: Linux + - type: Phone + browser: MobileIron + os: iOS + - type: Phone + browser: charlotte + os: Android + - type: Phone + browser: BlackBerry WebKit + os: BlackBerry + - type: Phone + browser: YandexSearch + os: Android + - type: Phone + browser: Salesforce + os: iOS + diff --git a/modules/ingest-user-agent/src/test/resources/other-devices.yml b/modules/ingest-user-agent/src/test/resources/other-devices.yml new file mode 100644 index 0000000000000..98595d961552b --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/other-devices.yml @@ -0,0 +1,22 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +robot_devices: + - type: Desktop + os: Tizen + browser: AppleWebKit + diff --git a/modules/ingest-user-agent/src/test/resources/robot-devices.yml b/modules/ingest-user-agent/src/test/resources/robot-devices.yml new file mode 100644 index 0000000000000..e3f2d0cedc10f --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/robot-devices.yml @@ -0,0 +1,123 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +robot_devices: + - type: Robot + os: CentOS + browser: AdsBot-Naver + - type: Robot + browser: iSec_Bot + os: Cloud + - type: Robot + browser: moatbot + os: Cloud + - type: Robot + browser: Baiduspider-render + os: Cloud + - type: Robot + browser: AhrefsBot + os: Cloud + - type: Robot + browser: Applebot + os: Cloud + - type: Robot + browser: Seekport Crawler + os: Cloud + - type: Robot + browser: Linespider + os: Cloud + - type: Robot + browser: pingbot + os: Cloud + - type: Robot + browser: YisouSpider + os: Cloud + - type: Robot + browser: HubSpot Crawler + os: Cloud + - type: Robot + browser: AdsBot + os: Cloud + - type: Robot + browser: net/bot + os: Cloud + - type: Robot + browser: Investment Crawler + os: Cloud + - type: Robot + browser: Bytespider + os: Cloud + - type: Robot + browser: IBM-Crawler + os: Cloud + - type: Robot + browser: BublupBot + os: Cloud + - type: Robot + browser: AppEngine-Google + os: Google Cloud + - type: Robot + browser: YandexBot + os: Cloud + - type: Robot + browser: Slackbot-LinkExpanding + os: Cloud + - type: Robot + browser: WebPageTest.org bot + os: Cloud + - type: Robot + browser: Baiduspider-image + os: Cloud + - type: Robot + browser: Pinterestbot + os: Cloud + - type: Robot + browser: YandexAccessibilityBot + os: Cloud + - type: Robot + browser: FacebookBot + os: Cloud + - type: Robot + browser: BLEXBot + os: Cloud + - type: Robot + browser: SuperBot + os: Cloud + - type: Robot + browser: Googlebot-News + os: Cloud + - type: Robot + browser: SMTBot + os: Cloud + - type: Robot + browser: GooglePlusBot + os: Cloud + - type: Robot + browser: niocBot + os: Cloud + - type: Robot + browser: SpiderWeb + os: Cloud + - type: Robot + browser: facebot + os: Cloud + - type: Robot + browser: MJ12bot + os: Cloud + - type: Robot + browser: ethical-bugbot + os: Linux diff --git a/modules/ingest-user-agent/src/test/resources/tablet-devices.yml b/modules/ingest-user-agent/src/test/resources/tablet-devices.yml new file mode 100644 index 0000000000000..75e6b222c639f --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/tablet-devices.yml @@ -0,0 +1,29 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tablet_devices: + - type: Tablet + os: BlackBerry Tablet OS + browser: Edg + - type: Tablet + browser: Amazon Silk + os: FireOS + - type: Tablet + browser: Crosswalk + os: Android + + From 4dc6427e10b74f467d5443128f1d3ad567b0b08e Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 22 Feb 2021 11:08:24 +0100 Subject: [PATCH 02/18] update overload --- .../ingest/useragent/UserAgentParser.java | 12 ++++--- .../useragent/DeviceTypeParserTests.java | 36 +++++++++---------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index a6cc6e51ac89a..acb3460f9730a 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -162,10 +162,6 @@ List getDevicePatterns() { return devicePatterns; } - String getName() { - return name; - } - public Details parse(String agentString) { Details details = cache.get(name, agentString); @@ -226,6 +222,14 @@ static final class VersionedName { this.patch = patch; this.build = build; } + + VersionedName(String name) { + this.name = name; + this.major = null; + this.minor = null; + this.patch = null; + this.build = null; + } } /** diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index a720f63952dc1..4966297de4f43 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -88,9 +88,9 @@ public static void setupDeviceParser() throws IOException { @SuppressWarnings("unchecked") public void testMacDesktop() throws Exception { - VersionedName os = new VersionedName("Mac OS X", null, null, null, null); + VersionedName os = new VersionedName("Mac OS X"); - VersionedName userAgent = new VersionedName("Chrome", null, null, null, null); + VersionedName userAgent = new VersionedName("Chrome"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -100,9 +100,9 @@ public void testMacDesktop() throws Exception { @SuppressWarnings("unchecked") public void testAndroidMobile() throws Exception { - VersionedName os = new VersionedName("iOS", null, null, null, null); + VersionedName os = new VersionedName("iOS"); - VersionedName userAgent = new VersionedName("Safari", null, null, null, null); + VersionedName userAgent = new VersionedName("Safari"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -112,11 +112,11 @@ public void testAndroidMobile() throws Exception { @SuppressWarnings("unchecked") public void testIPadTablet() throws Exception { - VersionedName os = new VersionedName("iOS", null, null, null, null); + VersionedName os = new VersionedName("iOS"); - VersionedName userAgent = new VersionedName("Safari", null, null, null, null); + VersionedName userAgent = new VersionedName("Safari"); - VersionedName device = new VersionedName("iPad", null, null, null, null); + VersionedName device = new VersionedName("iPad"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); @@ -126,9 +126,9 @@ public void testIPadTablet() throws Exception { @SuppressWarnings("unchecked") public void testWindowDesktop() throws Exception { - VersionedName os = new VersionedName("Mac OS X", null, null, null, null); + VersionedName os = new VersionedName("Mac OS X"); - VersionedName userAgent = new VersionedName("Chrome", null, null, null, null); + VersionedName userAgent = new VersionedName("Chrome"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -143,9 +143,9 @@ public void testRobotDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "robot_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + VersionedName os = new VersionedName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + VersionedName userAgent = new VersionedName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -161,12 +161,12 @@ public void testDesktopDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "desktop_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + VersionedName os = new VersionedName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + VersionedName userAgent = new VersionedName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); - if(deviceType.equals("Mobile")){ + if (deviceType.equals("Mobile")) { String name = "wow"; } assertThat(deviceType, is("Desktop")); @@ -181,9 +181,9 @@ public void testMobileDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "mobile_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + VersionedName os = new VersionedName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + VersionedName userAgent = new VersionedName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -199,9 +199,9 @@ public void testTabletDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "tablet_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os"), null, null, null, null); + VersionedName os = new VersionedName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser"), null, null, null, null); + VersionedName userAgent = new VersionedName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); From fd334b35c26e860eafe850dd366cf28cd68bc5aa Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 22 Feb 2021 13:52:11 +0100 Subject: [PATCH 03/18] fix license headers --- .../ingest/useragent/DeviceTypeParser.java | 21 +++++-------------- .../useragent/DeviceTypeParserTests.java | 21 +++++-------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index 5ef4a6dc1eb60..9f671edc7b401 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.ingest.useragent; diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index 4966297de4f43..b47a0cc9d6ef3 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.ingest.useragent; From d094f783f35b5e36ffd0cd5025124a17a6938949 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 22 Feb 2021 17:00:29 +0100 Subject: [PATCH 04/18] added more tests --- .../useragent/IngestUserAgentPlugin.java | 2 +- .../ingest/useragent/UserAgentParser.java | 11 ++++++++ .../ingest/useragent/UserAgentProcessor.java | 10 ++++--- .../useragent/UserAgentProcessorTests.java | 27 +++++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 0a52149af656b..8e917b9284069 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -68,7 +68,7 @@ static Map createUserAgentParsers(Path userAgentConfigD for (Path path : iterable) { String parserName = path.getFileName().toString(); try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream,regexStream, cache)); + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); } } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index 533fdd5b8102b..c8a876ed0e37f 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -50,6 +50,17 @@ final class UserAgentParser { } } + UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { + this.name = name; + this.cache = cache; + + try { + init(regexStream); + } catch (IOException e) { + throw new ElasticsearchParseException("error parsing regular expression file", e); + } + } + private void init(InputStream regexStream) throws IOException { // EMPTY is safe here because we don't use namedObject XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index b10a7a1fc229f..50f8e7090ee67 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -128,7 +128,11 @@ public IngestDocument execute(IngestDocument ingestDocument) { deviceDetails.put("type", uaClient.deviceType); } else { deviceDetails.put("name", "Other"); - deviceDetails.put("type", "Other"); + if (uaClient.deviceType != null) { + deviceDetails.put("type", uaClient.deviceType); + } else { + deviceDetails.put("type", "Other"); + } } uaDetails.put("device", deviceDetails); break; @@ -185,7 +189,7 @@ public UserAgentProcessor create(Map factories, Strin UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { throw newConfigurationException(TYPE, processorTag, - "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); + "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); } final Set properties; @@ -219,7 +223,7 @@ public static Property parseProperty(String propertyName) { return valueOf(propertyName.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + - Arrays.toString(EnumSet.allOf(Property.class).toArray())); + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } } } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index a6b46d68b5522..f157c35c7056a 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -108,6 +108,33 @@ public void testCommonBrowser() throws Exception { assertThat(target.get("device"), is(device)); } + @SuppressWarnings("unchecked") + public void testWIndowsOS() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Chrome")); + assertThat(target.get("version"), is("87.0.4280.141")); + + Map os = new HashMap<>(); + os.put("name", "Windows"); + os.put("version", "10"); + os.put("full", "Windows 10"); + assertThat(target.get("os"), is(os)); + Map device = new HashMap<>(); + device.put("name", "Other"); + device.put("type", "Desktop"); + assertThat(target.get("device"), is(device)); + } + @SuppressWarnings("unchecked") public void testUncommonDevice() throws Exception { Map document = new HashMap<>(); From 97d220a7c3431bcdf9a7b2317b18d138063372d0 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Tue, 23 Feb 2021 16:41:29 +0100 Subject: [PATCH 05/18] update pattern --- .../src/main/resources/device_type_regexes.yml | 2 +- .../ingest-user-agent/src/test/resources/desktop-devices.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml index a96ef173d2489..5204a59b45f1a 100644 --- a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml +++ b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml @@ -41,7 +41,7 @@ browser_parsers: - regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)' replacement: 'Desktop' # Mobile Browsers, Most Common - - regex: '^(Chrome Mobile|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' + - regex: '^(Chrome Mobile$|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' replacement: 'Mobile' # Mobile Browsers, Not Common - regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)' diff --git a/modules/ingest-user-agent/src/test/resources/desktop-devices.yml b/modules/ingest-user-agent/src/test/resources/desktop-devices.yml index 90c12d564da95..a23d44b984fff 100644 --- a/modules/ingest-user-agent/src/test/resources/desktop-devices.yml +++ b/modules/ingest-user-agent/src/test/resources/desktop-devices.yml @@ -172,3 +172,6 @@ desktop_devices: - type: Desktop browser: Iceweasel os: Linux + - type: Desktop + browser: Chrome Mobile WebView + os: Linux From f05d568dddf85216057bb40b3c02e6f1ef916677 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Wed, 24 Feb 2021 00:41:19 +0100 Subject: [PATCH 06/18] update formatting --- .../ingest/useragent/DeviceTypeParser.java | 31 +++-- .../ingest/useragent/UserAgentParser.java | 121 +++++++++--------- .../main/resources/device_type_regexes.yml | 17 ++- .../useragent/DeviceTypeParserTests.java | 14 +- .../src/test/resources/tablet-devices.yml | 14 +- 5 files changed, 120 insertions(+), 77 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index 9f671edc7b401..c1e13f762dd3c 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -9,7 +9,11 @@ package org.elasticsearch.ingest.useragent; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.InputStream; @@ -21,18 +25,19 @@ import java.util.regex.Pattern; import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; +import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; public class DeviceTypeParser { private static final String OS_PARSERS = "os_parsers"; private static final String BROWSER_PARSER = "browser_parsers"; private static final String DEVICE_PARSER = "device_parsers"; + private static final String AGENT_STRING_PARSER = "agent_string_parsers"; - private final List patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER); + private final List patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER, AGENT_STRING_PARSER); private final HashMap> deviceTypePatterns = new HashMap<>(); - public void init(InputStream regexStream) throws IOException { // EMPTY is safe here because we don't use namedObject XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, @@ -62,16 +67,22 @@ public void init(InputStream regexStream) throws IOException { } } - public String findDeviceType(UserAgentParser.VersionedName userAgent, - UserAgentParser.VersionedName os, UserAgentParser.VersionedName device) { + public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) { + + if (agentString != null) { + String deviceType = findMatch(deviceTypePatterns.get(AGENT_STRING_PARSER), agentString); + if (deviceType != null) { + return deviceType; + } + } + return findDeviceType(userAgent, os, device); + } + + public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) { ArrayList extractedDeviceTypes = new ArrayList<>(); String robot = "Robot", tablet = "Tablet", desktop = "Desktop", mobile = "Mobile"; - if (userAgent == null) { - return null; - } - for (String patternKey : patternListKeys) { String deviceType = null; switch (patternKey) { @@ -81,7 +92,7 @@ public String findDeviceType(UserAgentParser.VersionedName userAgent, } break; case BROWSER_PARSER: - if (userAgent.name != null) { + if (userAgent != null && userAgent.name != null) { deviceType = findMatch(deviceTypePatterns.get(patternKey), userAgent.name); } break; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index c8a876ed0e37f..5c25c47592a25 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; - import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -24,7 +23,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; - final class UserAgentParser { private final UserAgentCache cache; @@ -34,8 +32,6 @@ final class UserAgentParser { private final List uaPatterns = new ArrayList<>(); private final List osPatterns = new ArrayList<>(); private final List devicePatterns = new ArrayList<>(); - - private final String name; UserAgentParser(String name, InputStream regexStream, InputStream deviceTypeRegexStream, UserAgentCache cache) { @@ -77,23 +73,25 @@ private void init(InputStream regexStream) throws IOException { for (Map map : parserConfigurations) { uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), - map.get("v3_replacement"), map.get("v4_replacement"))); + map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), + map.get("v3_replacement"), map.get("v4_replacement"))); } - } else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { List> parserConfigurations = readParserConfigurations(yamlParser); for (Map map : parserConfigurations) { osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), - map.get("os_v3_replacement"), map.get("os_v4_replacement"))); + map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), + map.get("os_v3_replacement"), map.get("os_v4_replacement"))); } - } else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { List> parserConfigurations = readParserConfigurations(yamlParser); for (Map map : parserConfigurations) { devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), - map.get("device_replacement"), null, null, null, null)); + map.get("device_replacement"), null, null, null, null)); } } } @@ -113,7 +111,7 @@ private Pattern compilePattern(String regex, String regex_flag) { } } - static public List> readParserConfigurations(XContentParser yamlParser) throws IOException { + public static List> readParserConfigurations(XContentParser yamlParser) throws IOException { List> patternList = new ArrayList<>(); XContentParser.Token token = yamlParser.nextToken(); @@ -169,7 +167,7 @@ public Details parse(String agentString) { VersionedName userAgent = findMatch(uaPatterns, agentString); VersionedName operatingSystem = findMatch(osPatterns, agentString); VersionedName device = findMatch(devicePatterns, agentString); - String deviceType = deviceTypeParser.findDeviceType(userAgent, operatingSystem, device); + String deviceType = deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device); details = new Details(userAgent, operatingSystem, device, deviceType); @@ -214,7 +212,6 @@ static final class VersionedName { public final String patch; public final String build; - VersionedName(String name, String major, String minor, String patch, String build) { this.name = name; this.major = major; @@ -240,60 +237,60 @@ static final class UserAgentSubpattern { private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement; UserAgentSubpattern(Pattern pattern, String nameReplacement, - String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { - this.pattern = pattern; - this.nameReplacement = nameReplacement; - this.v1Replacement = v1Replacement; - this.v2Replacement = v2Replacement; - this.v3Replacement = v3Replacement; - this.v4Replacement = v4Replacement; + String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + this.v1Replacement = v1Replacement; + this.v2Replacement = v2Replacement; + this.v3Replacement = v3Replacement; + this.v4Replacement = v4Replacement; } public VersionedName match(String agentString) { - String name = null, major = null, minor = null, patch = null, build = null; - Matcher matcher = pattern.matcher(agentString); + String name = null, major = null, minor = null, patch = null, build = null; + Matcher matcher = pattern.matcher(agentString); - if (matcher.find() == false) { - return null; - } + if (matcher.find() == false) { + return null; + } - int groupCount = matcher.groupCount(); + int groupCount = matcher.groupCount(); - if (nameReplacement != null) { - if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { - name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); - } else { - name = nameReplacement; - } - } else if (groupCount >= 1) { - name = matcher.group(1); + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; } - - if (v1Replacement != null) { - major = v1Replacement; - } else if (groupCount >= 2) { - major = matcher.group(2); - } - - if (v2Replacement != null) { - minor = v2Replacement; - } else if (groupCount >= 3) { - minor = matcher.group(3); - } - - if (v3Replacement != null) { - patch = v3Replacement; - } else if (groupCount >= 4) { - patch = matcher.group(4); - } - - if (v4Replacement != null) { - build = v4Replacement; - } else if (groupCount >= 5) { - build = matcher.group(5); - } - - return name == null ? null : new VersionedName(name, major, minor, patch, build); + } else if (groupCount >= 1) { + name = matcher.group(1); + } + + if (v1Replacement != null) { + major = v1Replacement; + } else if (groupCount >= 2) { + major = matcher.group(2); + } + + if (v2Replacement != null) { + minor = v2Replacement; + } else if (groupCount >= 3) { + minor = matcher.group(3); + } + + if (v3Replacement != null) { + patch = v3Replacement; + } else if (groupCount >= 4) { + patch = matcher.group(4); + } + + if (v4Replacement != null) { + build = v4Replacement; + } else if (groupCount >= 5) { + build = matcher.group(5); + } + + return name == null ? null : new VersionedName(name, major, minor, patch, build); } - } + } } diff --git a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml index 5204a59b45f1a..0d606ceb86932 100644 --- a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml +++ b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml @@ -28,7 +28,7 @@ os_parsers: - regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)' replacement: 'Mobile' # Desktop OS, Not Common - - regex: '^(FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' + - regex: '^(Windows XP|Windows 7|Windows 10|FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' replacement: 'Desktop' - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' replacement: 'Tablet' @@ -50,5 +50,18 @@ browser_parsers: replacement: 'Tablet' device_parsers: - - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk|Kindle' + replacement: 'Tablet' + # Samsung tablets + - regex: 'SM-T\d+|SM-P\d+|GT-P\d+' replacement: 'Tablet' + # other tablets + - regex: 'Asus Nexus \d+|Lenovo TB' + replacement: 'Tablet' + +agent_string_parsers: + - regex: 'Synthetic|Scanner|Crawler|Site24x7|PagePeeker|SpeedCurve|RuxitSynthetic|Google Web Preview|Synthetic|SiteChecker|Parser' + replacement: 'Robot' + - regex: 'Tablet' + replacement: 'Tablet' + diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index b47a0cc9d6ef3..9a2b87848c809 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -53,6 +53,7 @@ private ArrayList> readTestDevices(InputStream regexStre testDevice.put("type", map.get("type")); testDevice.put("os", map.get("os")); testDevice.put("browser", map.get("browser")); + testDevice.put("device", map.get("device")); testDevices.add(testDevice); } @@ -124,6 +125,15 @@ public void testWindowDesktop() throws Exception { assertThat(deviceType, is("Desktop")); } + @SuppressWarnings("unchecked") + public void testRobotAgentString() throws Exception { + + String deviceType = deviceTypeParser.findDeviceType("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0.247) Gecko/20100101 Firefox/63.0.247 Site24x7", + null, null, null); + + assertThat(deviceType, is("Robot")); + } + @SuppressWarnings("unchecked") public void testRobotDevices() throws Exception { @@ -192,7 +202,9 @@ public void testTabletDevices() throws Exception { VersionedName userAgent = new VersionedName(testDevice.get("browser")); - String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + VersionedName device = new VersionedName(testDevice.get("device")); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); assertThat(deviceType, is("Tablet")); } diff --git a/modules/ingest-user-agent/src/test/resources/tablet-devices.yml b/modules/ingest-user-agent/src/test/resources/tablet-devices.yml index 75e6b222c639f..0f67c361d7c0b 100644 --- a/modules/ingest-user-agent/src/test/resources/tablet-devices.yml +++ b/modules/ingest-user-agent/src/test/resources/tablet-devices.yml @@ -25,5 +25,15 @@ tablet_devices: - type: Tablet browser: Crosswalk os: Android - - + - type: Tablet + browser: Chrome Mobile WebView + os: Android + device: Samsung SM-T590 + - type: Tablet + browser: Amazon Silk + os: Linux + device: Kindle + - type: Tablet + browser: Chrome + os: Android + device: Samsung SM-T307U From 4ee800fce9d55d295a76db77458e6bb284ad0988 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Wed, 24 Feb 2021 21:46:22 +0100 Subject: [PATCH 07/18] update formatting --- .../ingest/useragent/DeviceTypeParserTests.java | 14 +++++++++----- .../ingest/useragent/UserAgentProcessorTests.java | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index 9a2b87848c809..57e1fa743b85e 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -7,9 +7,13 @@ */ package org.elasticsearch.ingest.useragent; - -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; + import org.junit.BeforeClass; import java.io.IOException; @@ -24,7 +28,7 @@ import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.is; public class DeviceTypeParserTests extends ESTestCase { @@ -128,8 +132,8 @@ public void testWindowDesktop() throws Exception { @SuppressWarnings("unchecked") public void testRobotAgentString() throws Exception { - String deviceType = deviceTypeParser.findDeviceType("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0.247) Gecko/20100101 Firefox/63.0.247 Site24x7", - null, null, null); + String deviceType = deviceTypeParser.findDeviceType( + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0.247) Gecko/20100101 Firefox/63.0.247 Site24x7", null, null, null); assertThat(deviceType, is("Robot")); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index f157c35c7056a..99f84dea5eb86 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -192,7 +192,8 @@ public void testSpider() throws Exception { public void testTablet() throws Exception { Map document = new HashMap<>(); document.put("source_field", - "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Mobile/15E148 Safari/604.1"); + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/12.1 Mobile/15E148 Safari/604.1"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); processor.execute(ingestDocument); From 78c238fe0d8086dc88c328e4b5978bdc4a4f024f Mon Sep 17 00:00:00 2001 From: Shahzad Date: Wed, 24 Feb 2021 21:49:17 +0100 Subject: [PATCH 08/18] more formatting --- .../elasticsearch/ingest/useragent/IngestUserAgentPlugin.java | 4 ++-- .../elasticsearch/ingest/useragent/UserAgentProcessor.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 8e917b9284069..55f5e3862d037 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -28,7 +28,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin { private final Setting CACHE_SIZE_SETTING = Setting.longSetting("ingest.user_agent.cache_size", 1000, 0, - Setting.Property.NodeScope); + Setting.Property.NodeScope); static final String DEFAULT_PARSER_NAME = "_default_"; @@ -63,7 +63,7 @@ static Map createUserAgentParsers(Path userAgentConfigD PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yml"); try (Stream regexFiles = Files.find(userAgentConfigDirectory, 1, - (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { + (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { Iterable iterable = regexFiles::iterator; for (Path path : iterable) { String parserName = path.getFileName().toString(); diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 50f8e7090ee67..4f679f68aca2c 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -189,7 +189,7 @@ public UserAgentProcessor create(Map factories, Strin UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { throw newConfigurationException(TYPE, processorTag, - "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); + "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); } final Set properties; @@ -223,7 +223,7 @@ public static Property parseProperty(String propertyName) { return valueOf(propertyName.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + - Arrays.toString(EnumSet.allOf(Property.class).toArray())); + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } } } From 308af7f86e2c562798636d8b567cd01c88e4fdce Mon Sep 17 00:00:00 2001 From: Shahzad Date: Sat, 27 Feb 2021 21:45:14 +0100 Subject: [PATCH 09/18] fix test --- .../elasticsearch/ingest/useragent/DeviceTypeParser.java | 9 ++++++++- .../test/ingest-useragent/20_useragent_processor.yml | 2 +- .../test/ingest-useragent/30_custom_regex.yml | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index c1e13f762dd3c..88db82597a765 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -68,7 +68,9 @@ public void init(InputStream regexStream) throws IOException { } public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) { - + if(deviceTypePatterns.isEmpty()){ + return null; + } if (agentString != null) { String deviceType = findMatch(deviceTypePatterns.get(AGENT_STRING_PARSER), agentString); if (deviceType != null) { @@ -79,6 +81,11 @@ public String findDeviceType(String agentString, VersionedName userAgent, Versio } public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) { + + if(deviceTypePatterns.isEmpty()){ + return null; + } + ArrayList extractedDeviceTypes = new ArrayList<>(); String robot = "Robot", tablet = "Tablet", desktop = "Desktop", mobile = "Mobile"; diff --git a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 4daef7da0ce0a..a5e8e0b2490f9 100644 --- a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -32,7 +32,7 @@ - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } - match: { _source.user_agent.version: "33.0.1750.149" } - - match: { _source.user_agent.device: {"name": "Mac" }} + - match: { _source.user_agent.device: {"name": "Mac", type: "Desktop" }} --- "Test user agent processor with parameters": diff --git a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 763bea0ee4da0..cc7a461a338a1 100644 --- a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -30,6 +30,6 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } - - match: { _source.user_agent.device: {"name": "Other" }} + - match: { _source.user_agent.device: {"name": "Other", "type": "Other" }} - is_false: _source.user_agent.os - is_false: _source.user_agent.version From 4a669562a07ae4f5b8a13a9eff752360077befff Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 1 Mar 2021 09:22:25 +0100 Subject: [PATCH 10/18] update test --- docs/reference/ingest/processors/user-agent.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index b2a45b6395331..690bf3a98cc7d 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -69,7 +69,8 @@ Which returns "full": "Mac OS X 10.10.5" }, "device" : { - "name" : "Mac" + "name" : "Mac", + "type" : "Desktop" }, } } From d373587695e2a596a47ecbb993d6d867abf75b4a Mon Sep 17 00:00:00 2001 From: Shahzad Date: Fri, 5 Mar 2021 13:28:15 +0100 Subject: [PATCH 11/18] rename mobile dev type to phone, since mobile is more generic term --- .../ingest/useragent/DeviceTypeParser.java | 6 +++--- .../src/main/resources/device_type_regexes.yml | 12 ++++++------ .../ingest/useragent/DeviceTypeParserTests.java | 8 +++----- .../ingest/useragent/UserAgentProcessorTests.java | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index 88db82597a765..a7670ff7ea8f1 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -88,7 +88,7 @@ public String findDeviceType(VersionedName userAgent, VersionedName os, Versione ArrayList extractedDeviceTypes = new ArrayList<>(); - String robot = "Robot", tablet = "Tablet", desktop = "Desktop", mobile = "Mobile"; + String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone"; for (String patternKey : patternListKeys) { String deviceType = null; @@ -124,8 +124,8 @@ public String findDeviceType(VersionedName userAgent, VersionedName os, Versione if (extractedDeviceTypes.contains(tablet)) { return tablet; } - if (extractedDeviceTypes.contains(mobile)) { - return mobile; + if (extractedDeviceTypes.contains(phone)) { + return phone; } if (extractedDeviceTypes.contains(desktop)) { return desktop; diff --git a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml index 0d606ceb86932..88a860a9d9d83 100644 --- a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml +++ b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml @@ -24,9 +24,9 @@ os_parsers: # Desktop OS, Most Common - regex: '^(Windows$|Windows NT$|Mac OS X|Linux$|Chrome OS|Fedora$|Ubuntu$)' replacement: 'Desktop' - # Mobile OS + # Phone OS - regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)' - replacement: 'Mobile' + replacement: 'Phone' # Desktop OS, Not Common - regex: '^(Windows XP|Windows 7|Windows 10|FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' replacement: 'Desktop' @@ -40,12 +40,12 @@ browser_parsers: # Desktop Browsers - regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)' replacement: 'Desktop' - # Mobile Browsers, Most Common + # Phone Browsers, Most Common - regex: '^(Chrome Mobile$|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' - replacement: 'Mobile' - # Mobile Browsers, Not Common + replacement: 'Phone' + # Phone Browsers, Not Common - regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)' - replacement: 'Mobile' + replacement: 'Phone' - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' replacement: 'Tablet' diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index 57e1fa743b85e..439ee3a034d7f 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -100,7 +100,7 @@ public void testAndroidMobile() throws Exception { String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); - assertThat(deviceType, is("Mobile")); + assertThat(deviceType, is("Phone")); } @SuppressWarnings("unchecked") @@ -169,9 +169,7 @@ public void testDesktopDevices() throws Exception { VersionedName userAgent = new VersionedName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); - if (deviceType.equals("Mobile")) { - String name = "wow"; - } + assertThat(deviceType, is("Desktop")); } } @@ -190,7 +188,7 @@ public void testMobileDevices() throws Exception { String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); - assertThat(deviceType, is("Mobile")); + assertThat(deviceType, is("Phone")); } } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 99f84dea5eb86..4e37396bded25 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -160,7 +160,7 @@ public void testUncommonDevice() throws Exception { Map device = new HashMap<>(); device.put("name", "Motorola Xoom"); - device.put("type", "Mobile"); + device.put("type", "Phone"); assertThat(target.get("device"), is(device)); } From 4ba9a4a604913b3043be419a639ddb7dc0963113 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Fri, 12 Mar 2021 17:13:53 +0100 Subject: [PATCH 12/18] PR feedback, formatting and remove extra constructors --- .../ingest/useragent/DeviceTypeParser.java | 7 ++-- .../useragent/IngestUserAgentPlugin.java | 11 ++++- .../ingest/useragent/UserAgentParser.java | 32 ++++----------- .../useragent/DeviceTypeParserTests.java | 40 ++++++++++--------- .../useragent/UserAgentProcessorTests.java | 4 +- 5 files changed, 42 insertions(+), 52 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index a7670ff7ea8f1..233bf67d5062c 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -33,6 +33,7 @@ public class DeviceTypeParser { private static final String BROWSER_PARSER = "browser_parsers"; private static final String DEVICE_PARSER = "device_parsers"; private static final String AGENT_STRING_PARSER = "agent_string_parsers"; + private static final String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone"; private final List patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER, AGENT_STRING_PARSER); @@ -68,7 +69,7 @@ public void init(InputStream regexStream) throws IOException { } public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) { - if(deviceTypePatterns.isEmpty()){ + if (deviceTypePatterns.isEmpty()) { return null; } if (agentString != null) { @@ -82,14 +83,12 @@ public String findDeviceType(String agentString, VersionedName userAgent, Versio public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) { - if(deviceTypePatterns.isEmpty()){ + if (deviceTypePatterns.isEmpty()) { return null; } ArrayList extractedDeviceTypes = new ArrayList<>(); - String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone"; - for (String patternKey : patternListKeys) { String deviceType = null; switch (patternKey) { diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 55f5e3862d037..8887f8de8e565 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -13,6 +13,7 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -53,10 +54,16 @@ public Map getProcessors(Processor.Parameters paramet static Map createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { Map userAgentParsers = new HashMap<>(); + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); + boolean checkDeviceTypeFile = new File(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml").exists(); + + if (checkDeviceTypeFile == false) { + deviceTypeRegexStream = null; + } UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), - IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"), cache); + deviceTypeRegexStream, cache); userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { @@ -68,7 +75,7 @@ static Map createUserAgentParsers(Path userAgentConfigD for (Path path : iterable) { String parserName = path.getFileName().toString(); try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStream, cache)); } } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index 5c25c47592a25..bc01f722998b1 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -26,9 +26,7 @@ final class UserAgentParser { private final UserAgentCache cache; - private final DeviceTypeParser deviceTypeParser = new DeviceTypeParser(); - private final List uaPatterns = new ArrayList<>(); private final List osPatterns = new ArrayList<>(); private final List devicePatterns = new ArrayList<>(); @@ -40,18 +38,9 @@ final class UserAgentParser { try { init(regexStream); - this.deviceTypeParser.init(deviceTypeRegexStream); - } catch (IOException e) { - throw new ElasticsearchParseException("error parsing regular expression file", e); - } - } - - UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { - this.name = name; - this.cache = cache; - - try { - init(regexStream); + if (deviceTypeRegexStream != null) { + deviceTypeParser.init(deviceTypeRegexStream); + } } catch (IOException e) { throw new ElasticsearchParseException("error parsing regular expression file", e); } @@ -160,6 +149,10 @@ List getDevicePatterns() { return devicePatterns; } + String getName() { + return name; + } + public Details parse(String agentString) { Details details = cache.get(name, agentString); @@ -168,10 +161,7 @@ public Details parse(String agentString) { VersionedName operatingSystem = findMatch(osPatterns, agentString); VersionedName device = findMatch(devicePatterns, agentString); String deviceType = deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device); - - details = new Details(userAgent, operatingSystem, device, deviceType); - cache.put(name, agentString, details); } @@ -219,14 +209,6 @@ static final class VersionedName { this.patch = patch; this.build = build; } - - VersionedName(String name) { - this.name = name; - this.major = null; - this.minor = null; - this.patch = null; - this.build = null; - } } /** diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index 439ee3a034d7f..e98a8b13424b9 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -34,7 +34,6 @@ public class DeviceTypeParserTests extends ESTestCase { private static DeviceTypeParser deviceTypeParser; - private ArrayList> readTestDevices(InputStream regexStream, String keyName) throws IOException { XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, regexStream); @@ -68,6 +67,9 @@ private ArrayList> readTestDevices(InputStream regexStre return testDevices; } + private static VersionedName getVersionName(String name){ + return new VersionedName(name, null, null, null, null); + } @BeforeClass public static void setupDeviceParser() throws IOException { @@ -82,9 +84,9 @@ public static void setupDeviceParser() throws IOException { @SuppressWarnings("unchecked") public void testMacDesktop() throws Exception { - VersionedName os = new VersionedName("Mac OS X"); + VersionedName os = getVersionName("Mac OS X"); - VersionedName userAgent = new VersionedName("Chrome"); + VersionedName userAgent = getVersionName("Chrome"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -94,9 +96,9 @@ public void testMacDesktop() throws Exception { @SuppressWarnings("unchecked") public void testAndroidMobile() throws Exception { - VersionedName os = new VersionedName("iOS"); + VersionedName os = getVersionName("iOS"); - VersionedName userAgent = new VersionedName("Safari"); + VersionedName userAgent = getVersionName("Safari"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -106,11 +108,11 @@ public void testAndroidMobile() throws Exception { @SuppressWarnings("unchecked") public void testIPadTablet() throws Exception { - VersionedName os = new VersionedName("iOS"); + VersionedName os = getVersionName("iOS"); - VersionedName userAgent = new VersionedName("Safari"); + VersionedName userAgent = getVersionName("Safari"); - VersionedName device = new VersionedName("iPad"); + VersionedName device = getVersionName("iPad"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); @@ -120,9 +122,9 @@ public void testIPadTablet() throws Exception { @SuppressWarnings("unchecked") public void testWindowDesktop() throws Exception { - VersionedName os = new VersionedName("Mac OS X"); + VersionedName os = getVersionName("Mac OS X"); - VersionedName userAgent = new VersionedName("Chrome"); + VersionedName userAgent = getVersionName("Chrome"); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -146,9 +148,9 @@ public void testRobotDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "robot_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os")); + VersionedName os = getVersionName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser")); + VersionedName userAgent = getVersionName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -164,9 +166,9 @@ public void testDesktopDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "desktop_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os")); + VersionedName os = getVersionName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser")); + VersionedName userAgent = getVersionName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -182,9 +184,9 @@ public void testMobileDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "mobile_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os")); + VersionedName os = getVersionName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser")); + VersionedName userAgent = getVersionName(testDevice.get("browser")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); @@ -200,11 +202,11 @@ public void testTabletDevices() throws Exception { ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "tablet_devices"); for (HashMap testDevice : testDevices) { - VersionedName os = new VersionedName(testDevice.get("os")); + VersionedName os = getVersionName(testDevice.get("os")); - VersionedName userAgent = new VersionedName(testDevice.get("browser")); + VersionedName userAgent = getVersionName(testDevice.get("browser")); - VersionedName device = new VersionedName(testDevice.get("device")); + VersionedName device = getVersionName(testDevice.get("device")); String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 4e37396bded25..e2f28bfeff751 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -37,7 +37,7 @@ public static void setupProcessor() throws IOException { assertNotNull(regexStream); assertNotNull(deviceTypeRegexStream); - UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream,deviceTypeRegexStream, new UserAgentCache(1000)); + UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser, EnumSet.allOf(UserAgentProcessor.Property.class), false); @@ -109,7 +109,7 @@ public void testCommonBrowser() throws Exception { } @SuppressWarnings("unchecked") - public void testWIndowsOS() throws Exception { + public void testWindowsOS() throws Exception { Map document = new HashMap<>(); document.put("source_field", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"); From 30f9ab27012c7f4380592de9b2b50f24ed53d308 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 15 Mar 2021 09:42:30 +0100 Subject: [PATCH 13/18] remove usage of java.io.File --- .../ingest/useragent/IngestUserAgentPlugin.java | 5 ++--- .../org/elasticsearch/ingest/useragent/UserAgentParser.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 8887f8de8e565..e0becdf2d0d29 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -13,13 +13,13 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.StandardOpenOption; +import java.nio.file.Paths; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -55,9 +55,8 @@ public Map getProcessors(Processor.Parameters paramet static Map createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { Map userAgentParsers = new HashMap<>(); InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - boolean checkDeviceTypeFile = new File(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml").exists(); - if (checkDeviceTypeFile == false) { + if (Files.exists(Paths.get(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml")) == false) { deviceTypeRegexStream = null; } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index bc01f722998b1..56cd668d4ecf9 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -100,7 +100,7 @@ private Pattern compilePattern(String regex, String regex_flag) { } } - public static List> readParserConfigurations(XContentParser yamlParser) throws IOException { + static List> readParserConfigurations(XContentParser yamlParser) throws IOException { List> patternList = new ArrayList<>(); XContentParser.Token token = yamlParser.nextToken(); From c3e70615a623823c29501f3ed7c396a8e0ccf316 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 15 Mar 2021 10:25:40 +0100 Subject: [PATCH 14/18] more forbidden --- .../elasticsearch/ingest/useragent/IngestUserAgentPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index e0becdf2d0d29..55aab13aabd02 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -19,7 +19,7 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.StandardOpenOption; -import java.nio.file.Paths; +import org.elasticsearch.common.io.PathUtils; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -56,7 +56,7 @@ static Map createUserAgentParsers(Path userAgentConfigD Map userAgentParsers = new HashMap<>(); InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - if (Files.exists(Paths.get(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml")) == false) { + if (Files.exists(PathUtils.get(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml")) == false) { deviceTypeRegexStream = null; } From 56b1d3f9d31cd58671fec742d52bd0970a6d2b40 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Thu, 18 Mar 2021 09:05:08 +0100 Subject: [PATCH 15/18] test --- .../elasticsearch/ingest/useragent/IngestUserAgentPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 55aab13aabd02..5ea6da0eb7ae5 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -19,7 +19,6 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.StandardOpenOption; -import org.elasticsearch.common.io.PathUtils; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -56,7 +55,7 @@ static Map createUserAgentParsers(Path userAgentConfigD Map userAgentParsers = new HashMap<>(); InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - if (Files.exists(PathUtils.get(String.valueOf(userAgentConfigDirectory), "/device_type_regexes.yml")) == false) { + if (Files.exists(Path.of(userAgentConfigDirectory + "/device_type_regexes.yml")) == false) { deviceTypeRegexStream = null; } From fc5cb29f9de2ffa3cc8b39323a569814c6ced87f Mon Sep 17 00:00:00 2001 From: Shahzad Date: Thu, 18 Mar 2021 10:00:42 +0100 Subject: [PATCH 16/18] pass new stream for custom use case --- .../elasticsearch/ingest/useragent/DeviceTypeParser.java | 1 - .../ingest/useragent/IngestUserAgentPlugin.java | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java index 233bf67d5062c..0108e428ba451 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -144,7 +144,6 @@ private String findMatch(List possiblePatterns, String mat return null; } - static final class DeviceTypeSubPattern { private final Pattern pattern; private final String nameReplacement; diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 5ea6da0eb7ae5..3401eb81ae543 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -55,9 +55,6 @@ static Map createUserAgentParsers(Path userAgentConfigD Map userAgentParsers = new HashMap<>(); InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - if (Files.exists(Path.of(userAgentConfigDirectory + "/device_type_regexes.yml")) == false) { - deviceTypeRegexStream = null; - } UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), @@ -73,7 +70,8 @@ static Map createUserAgentParsers(Path userAgentConfigD for (Path path : iterable) { String parserName = path.getFileName().toString(); try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStream, cache)); + InputStream deviceTypeRegexStreamC = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStreamC, cache)); } } } From 2ea65979c74e5ece84de769b3352151891d29e2c Mon Sep 17 00:00:00 2001 From: Shahzad Date: Thu, 18 Mar 2021 15:59:34 +0100 Subject: [PATCH 17/18] update docs --- docs/reference/ingest/common-log-format-example.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc index ef46e7c444367..96b5d4fc95b6c 100644 --- a/docs/reference/ingest/common-log-format-example.asciidoc +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -186,7 +186,8 @@ GET my-index/_doc/1 }, "name": "Chrome", "device": { - "name": "Mac" + "name": "Mac", + "type": "Desktop" }, "version": "52.0.2743.116" } From 018cacb500cf3bdd84e6f2e2e2c590b3bbaeca1c Mon Sep 17 00:00:00 2001 From: Shahzad Date: Fri, 26 Mar 2021 20:42:58 +0100 Subject: [PATCH 18/18] update styling and naming --- .../ingest/useragent/IngestUserAgentPlugin.java | 10 ++++------ .../ingest/useragent/DeviceTypeParserTests.java | 16 ++++++++-------- ...ktop-devices.yml => test-desktop-devices.yml} | 0 ...obile-devices.yml => test-mobile-devices.yml} | 0 ...{other-devices.yml => test-other-devices.yml} | 0 ...{robot-devices.yml => test-robot-devices.yml} | 0 ...ablet-devices.yml => test-tablet-devices.yml} | 0 7 files changed, 12 insertions(+), 14 deletions(-) rename modules/ingest-user-agent/src/test/resources/{desktop-devices.yml => test-desktop-devices.yml} (100%) rename modules/ingest-user-agent/src/test/resources/{mobile-devices.yml => test-mobile-devices.yml} (100%) rename modules/ingest-user-agent/src/test/resources/{other-devices.yml => test-other-devices.yml} (100%) rename modules/ingest-user-agent/src/test/resources/{robot-devices.yml => test-robot-devices.yml} (100%) rename modules/ingest-user-agent/src/test/resources/{tablet-devices.yml => test-tablet-devices.yml} (100%) diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index 3401eb81ae543..dfaed02a2323e 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -53,12 +53,10 @@ public Map getProcessors(Processor.Parameters paramet static Map createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { Map userAgentParsers = new HashMap<>(); - InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), - deviceTypeRegexStream, cache); + IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"), cache); userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { @@ -69,9 +67,9 @@ static Map createUserAgentParsers(Path userAgentConfigD Iterable iterable = regexFiles::iterator; for (Path path : iterable) { String parserName = path.getFileName().toString(); - try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - InputStream deviceTypeRegexStreamC = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"); - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStreamC, cache)); + try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ); + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml")) { + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStream, cache)); } } } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java index e98a8b13424b9..dcde87fbb258e 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -143,9 +143,9 @@ public void testRobotAgentString() throws Exception { @SuppressWarnings("unchecked") public void testRobotDevices() throws Exception { - InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/robot-devices.yml"); + InputStream testRobotDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-robot-devices.yml"); - ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "robot_devices"); + ArrayList> testDevices = readTestDevices(testRobotDevices, "robot_devices"); for (HashMap testDevice : testDevices) { VersionedName os = getVersionName(testDevice.get("os")); @@ -161,9 +161,9 @@ public void testRobotDevices() throws Exception { @SuppressWarnings("unchecked") public void testDesktopDevices() throws Exception { - InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/desktop-devices.yml"); + InputStream testDesktopDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-desktop-devices.yml"); - ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "desktop_devices"); + ArrayList> testDevices = readTestDevices(testDesktopDevices, "desktop_devices"); for (HashMap testDevice : testDevices) { VersionedName os = getVersionName(testDevice.get("os")); @@ -179,9 +179,9 @@ public void testDesktopDevices() throws Exception { @SuppressWarnings("unchecked") public void testMobileDevices() throws Exception { - InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/mobile-devices.yml"); + InputStream testMobileDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-mobile-devices.yml"); - ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "mobile_devices"); + ArrayList> testDevices = readTestDevices(testMobileDevices, "mobile_devices"); for (HashMap testDevice : testDevices) { VersionedName os = getVersionName(testDevice.get("os")); @@ -197,9 +197,9 @@ public void testMobileDevices() throws Exception { @SuppressWarnings("unchecked") public void testTabletDevices() throws Exception { - InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/tablet-devices.yml"); + InputStream testTabletDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-tablet-devices.yml"); - ArrayList> testDevices = readTestDevices(deviceTypeRegexStream, "tablet_devices"); + ArrayList> testDevices = readTestDevices(testTabletDevices, "tablet_devices"); for (HashMap testDevice : testDevices) { VersionedName os = getVersionName(testDevice.get("os")); diff --git a/modules/ingest-user-agent/src/test/resources/desktop-devices.yml b/modules/ingest-user-agent/src/test/resources/test-desktop-devices.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/desktop-devices.yml rename to modules/ingest-user-agent/src/test/resources/test-desktop-devices.yml diff --git a/modules/ingest-user-agent/src/test/resources/mobile-devices.yml b/modules/ingest-user-agent/src/test/resources/test-mobile-devices.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/mobile-devices.yml rename to modules/ingest-user-agent/src/test/resources/test-mobile-devices.yml diff --git a/modules/ingest-user-agent/src/test/resources/other-devices.yml b/modules/ingest-user-agent/src/test/resources/test-other-devices.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/other-devices.yml rename to modules/ingest-user-agent/src/test/resources/test-other-devices.yml diff --git a/modules/ingest-user-agent/src/test/resources/robot-devices.yml b/modules/ingest-user-agent/src/test/resources/test-robot-devices.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/robot-devices.yml rename to modules/ingest-user-agent/src/test/resources/test-robot-devices.yml diff --git a/modules/ingest-user-agent/src/test/resources/tablet-devices.yml b/modules/ingest-user-agent/src/test/resources/test-tablet-devices.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/tablet-devices.yml rename to modules/ingest-user-agent/src/test/resources/test-tablet-devices.yml