ES|QL: Adding USER_AGENT command

docs/changelog/144384.yaml

@@ -0,0 +1,6 @@
+area: ES|QL
+issues:
+ - 134886
+pr: 144384
+summary: Adding ES|QL USER_AGENT command
+type: feature

docs/reference/query-languages/esql/_snippets/commands/examples/user_agent.csv-spec/basic.md

@@ -0,0 +1,11 @@
+% This is generated by ESQL's CommandDocsTests. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+ROW input = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"
+| USER_AGENT ua = input WITH { "extract_device_type": true }
+| KEEP ua.*
+```
+
+| ua.name:keyword | ua.version:keyword | ua.os.name:keyword | ua.os.version:keyword | ua.os.full:keyword | ua.device.name:keyword | ua.device.type:keyword |
+| --- | --- | --- | --- | --- | --- | --- |
+| Chrome | 33.0.1750.149 | Mac OS X | 10.9.2 | Mac OS X 10.9.2 | Mac | Desktop |

docs/reference/query-languages/esql/_snippets/commands/layout/user_agent.md

@@ -0,0 +1,117 @@
+```yaml {applies_to}
+serverless: preview
+stack: preview 9.4
+```
+
+The `USER_AGENT` processing command parses a user-agent string and extracts its components (name, version, OS, device) into new columns.
+
+::::{note}
+This command doesn't support multi-value inputs.
+::::
+
+
+## Syntax
+
+```esql
+USER_AGENT prefix = expression [WITH { option = value [, ...] }]
+```
+
+## Parameters
+
+`prefix`
+:   The prefix for the output columns. The extracted components are available as `prefix.component`.
+
+`expression`
+:   The string expression containing the user-agent string to parse.
+
+## WITH options
+
+`regex_file`
+:   The name of the parser configuration to use. Default: `_default_`, which uses the built-in regexes from [uap-core](https://github.com/ua-parser/uap-core). To use a custom regex file, place a `.yml` file in the `config/user-agent` directory on each node before starting Elasticsearch. The file must be present at node startup; changes or new files added while the node is running have no effect. Pass the filename (including the `.yml` extension) as the value. Custom regex files are typically variants of the default, either a more recent uap-core release or a customized version.
+
+`extract_device_type`
+:   When `true`, extracts device type (e.g., Desktop, Phone, Tablet) on a best-effort basis and includes `prefix.device.type` in the output. Default: `false`.
+
+`properties`
+:   List of property groups to include in the output. Each value expands to one or more columns: `name` → `prefix.name`; `version` → `prefix.version`; `os` → `prefix.os.name`, `prefix.os.version`, `prefix.os.full`; `device` → `prefix.device.name` (and `prefix.device.type` when `extract_device_type` is `true`). Default: `["name", "version", "os", "device"]`. You can pass a subset to reduce output columns.
+
+## Using a custom regex file
+
+To use a custom regex file instead of the built-in uap-core patterns:
+
+1. Place a `.yml` file in the `config/user-agent` directory on each node.
+2. Create the directory and file before starting Elasticsearch.
+3. Pass the filename (including the `.yml` extension) as the `regex_file` option.
+
+Files must be present at node startup. Changes to existing files or new files added while the node is running have no effect until the node is restarted.
+
+::::{note}
+Before version 9.4, this directory was named `config/ingest-user-agent`. The old directory name is still supported as a fallback but is deprecated.
+::::
+
+Custom regex files are typically variants of the default [uap-core regexes.yaml](https://github.com/ua-parser/uap-core/blob/master/regexes.yaml), either a more recent release or a customized version for specific user-agent patterns. Use a custom file when you need to support newer user-agent formats before they are available in the built-in patterns, or to parse specialized or non-standard user-agent strings.
+
+## Description
+
+The `USER_AGENT` command parses a user-agent string and extracts its parts into new columns.
+The new columns are prefixed with the specified `prefix` followed by a dot (`.`).
+
+This command is the query-time equivalent of the [User-Agent ingest processor](/reference/enrich-processor/user-agent-processor.md).
+
+The following columns may be created (depending on `properties` and `extract_device_type`):
+
+`prefix.name`
+:   The user-agent name (e.g., Chrome, Firefox).
+
+`prefix.version`
+:   The user-agent version.
+
+`prefix.os.name`
+:   The operating system name.
+
+`prefix.os.version`
+:   The operating system version.
+
+`prefix.os.full`
+:   The full operating system string.
+
+`prefix.device.name`
+:   The device name.
+
+`prefix.device.type`
+:   The device type (e.g., Desktop, Phone). Only present when `extract_device_type` is `true`.
+
+If a component is missing or the input is not a valid user-agent string, the corresponding column contains `null`.
+If the expression evaluates to `null` or blank, all output columns are `null`.
+
+## Examples
+
+The following example parses a user-agent string and extracts its parts:
+
+:::{include} ../examples/user_agent.csv-spec/basic.md
+:::
+
+To limit output to specific properties or include device type, use the `properties` and `extract_device_type` options:
+
+```esql
+ROW ua_str = "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15"
+| USER_AGENT ua = ua_str WITH { "properties": ["name", "version", "device"], "extract_device_type": true }
+| KEEP ua.*
+```
+
+To use a custom regex file (e.g. `my-regexes.yml` in `config/user-agent`), pass the filename including the extension:
+
+```esql
+FROM web_logs
+| USER_AGENT ua = user_agent WITH { "regex_file": "my-regexes.yml" }
+| KEEP ua.name, ua.version
+```
+
+You can use the extracted parts in subsequent commands, for example to filter by browser:
+
+```esql
+FROM web_logs
+| USER_AGENT ua = user_agent
+| WHERE ua.name == "Firefox"
+| STATS COUNT(*) BY ua.version
+```

docs/reference/query-languages/esql/_snippets/lists/processing-commands.md

@@ -23,5 +23,6 @@
 * [`SAMPLE`](/reference/query-languages/esql/commands/sample.md) {applies_to}`stack: preview 9.1` {applies_to}`serverless: preview`
 * [`SORT`](/reference/query-languages/esql/commands/sort.md)
 * [`STATS`](/reference/query-languages/esql/commands/stats-by.md)
+% * [`USER_AGENT`](/reference/query-languages/esql/commands/user-agent.md) {applies_to}`stack: preview 9.4` {applies_to}`serverless: preview`
 * [`URI_PARTS`](/reference/query-languages/esql/commands/uri-parts.md) {applies_to}`stack: preview 9.4` {applies_to}`serverless: preview`
 * [`WHERE`](/reference/query-languages/esql/commands/where.md)

docs/reference/query-languages/esql/commands/user-agent.md

@@ -0,0 +1,10 @@
+---
+navigation_title: "USER_AGENT"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-commands.html#esql-user_agent
+---
+
+# {{esql}} `USER_AGENT` command [esql-user_agent]
+
+:::{include} ../_snippets/commands/layout/user_agent.md
+:::

docs/reference/query-languages/toc.yml

@@ -131,6 +131,7 @@ toc:
                   - file: esql/commands/sample.md
                   - file: esql/commands/sort.md
                   - file: esql/commands/stats-by.md
+                  - hidden: esql/commands/user-agent.md
                   - file: esql/commands/uri-parts.md
                   - file: esql/commands/where.md
           - file: esql/esql-functions-operators.md

modules/user-agent/src/main/java/org/elasticsearch/useragent/UserAgentCache.java

@@ -20,13 +20,13 @@ class UserAgentCache {
         cache = CacheBuilder.<CompositeCacheKey, Details>builder().setMaximumWeight(cacheSize).build();
     }
 
-    public Details get(String parserName, String userAgent) {
-        return cache.get(new CompositeCacheKey(parserName, userAgent));
+    public Details get(String parserName, String userAgent, boolean extractDeviceType) {
+        return cache.get(new CompositeCacheKey(parserName, userAgent, extractDeviceType));
     }
 
-    public void put(String parserName, String userAgent, Details details) {
-        cache.put(new CompositeCacheKey(parserName, userAgent), details);
+    public void put(String parserName, String userAgent, Details details, boolean extractDeviceType) {
+        cache.put(new CompositeCacheKey(parserName, userAgent, extractDeviceType), details);
     }
 
-    private record CompositeCacheKey(String parserName, String userAgent) {}
+    private record CompositeCacheKey(String parserName, String userAgent, boolean extractDeviceType) {}
 }

modules/user-agent/src/main/java/org/elasticsearch/useragent/UserAgentParserImpl.java

@@ -183,7 +183,7 @@ String getName() {
 
     @Override
     public Details parseUserAgentInfo(String agentString, boolean extractDeviceType) {
-        Details details = cache.get(name, agentString);
+        Details details = cache.get(name, agentString, extractDeviceType);
         if (details == null) {
             VersionedName userAgent = findMatch(uaPatterns, agentString);
             VersionedName operatingSystem = findMatch(osPatterns, agentString);
@@ -199,7 +199,7 @@ public Details parseUserAgentInfo(String agentString, boolean extractDeviceType)
             VersionedName dev = (device != null && device.name() != null) ? device : null;
 
             details = new Details(uaName, uaVersion, os, osFull, dev, deviceType);
-            cache.put(name, agentString, details);
+            cache.put(name, agentString, details, extractDeviceType);
         }
         return details;
     }

modules/user-agent/src/main/java/org/elasticsearch/useragent/UserAgentParserRegistry.java → modules/user-agent/src/main/java/org/elasticsearch/useragent/UserAgentParserRegistryImpl.java

@@ -23,15 +23,15 @@
 import java.util.Map;
 import java.util.stream.Stream;
 
-public class UserAgentParserRegistry implements org.elasticsearch.useragent.api.UserAgentParserRegistry {
+class UserAgentParserRegistryImpl implements org.elasticsearch.useragent.api.UserAgentParserRegistry {
 
-    private static final Logger logger = LogManager.getLogger(UserAgentParserRegistry.class);
+    private static final Logger logger = LogManager.getLogger(UserAgentParserRegistryImpl.class);
 
     static final String DEFAULT_PARSER_NAME = org.elasticsearch.useragent.api.UserAgentParserRegistry.DEFAULT_PARSER_NAME;
 
     private final Map<String, UserAgentParserImpl> registry;
 
-    UserAgentParserRegistry(UserAgentCache cache, Path... regexFileDirectories) {
+    UserAgentParserRegistryImpl(UserAgentCache cache, Path... regexFileDirectories) {
         registry = createParsersMap(cache, regexFileDirectories);
     }
 

modules/user-agent/src/main/java/org/elasticsearch/useragent/UserAgentPlugin.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.UserAgentParserRegistryProvider;
+import org.elasticsearch.useragent.api.UserAgentParserRegistry;
 
 import java.nio.file.Path;
 import java.util.List;
@@ -37,7 +38,7 @@ public class UserAgentPlugin extends Plugin implements UserAgentParserRegistryPr
     );
 
     @Override
-    public org.elasticsearch.useragent.api.UserAgentParserRegistry createUserAgentParserRegistry(Environment env) {
+    public UserAgentParserRegistry createRegistry(Environment env) {
         return createRegistry(env, env.settings());
     }
 
@@ -56,6 +57,6 @@ public static UserAgentParserRegistry createRegistry(Environment env, Settings s
         Path ingestUserAgentConfigDirectory = env.configDir().resolve("ingest-user-agent");
         long cacheSize = CACHE_SIZE_SETTING.get(settings);
         UserAgentCache cache = new UserAgentCache(cacheSize);
-        return new UserAgentParserRegistry(cache, userAgentConfigDirectory, ingestUserAgentConfigDirectory);
+        return new UserAgentParserRegistryImpl(cache, userAgentConfigDirectory, ingestUserAgentConfigDirectory);
     }
 }