Skip to content

Commit 1849346

Browse files
astefanastefan
authored
EQL: Length function implementation (elastic#54209)
1 parent d90426a commit 1849346

File tree

12 files changed

+366
-4
lines changed

12 files changed

+366
-4
lines changed

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.eql.expression.function;
88

9+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
910
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
1011
import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
1112
import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
@@ -23,8 +24,9 @@ private static FunctionDefinition[][] functions() {
2324
// Scalar functions
2425
// String
2526
new FunctionDefinition[] {
26-
def(Substring.class, Substring::new, "substring"),
27-
},
27+
def(Length.class, Length::new, "length"),
28+
def(Substring.class, Substring::new, "substring")
29+
}
2830
};
2931
}
3032

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
8+
9+
import org.elasticsearch.xpack.ql.expression.Expression;
10+
import org.elasticsearch.xpack.ql.expression.Expressions;
11+
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
12+
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
13+
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
14+
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
15+
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
16+
import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
17+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
18+
import org.elasticsearch.xpack.ql.tree.Source;
19+
import org.elasticsearch.xpack.ql.type.DataType;
20+
import org.elasticsearch.xpack.ql.type.DataTypes;
21+
22+
import java.util.Arrays;
23+
import java.util.List;
24+
import java.util.Locale;
25+
26+
import static java.lang.String.format;
27+
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor.doProcess;
28+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
29+
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
30+
31+
/**
32+
* EQL specific length function acting on every type of field, not only strings.
33+
* For strings it will return the length of that specific string, for any other type it will return 0.
34+
*/
35+
public class Length extends ScalarFunction {
36+
37+
private final Expression source;
38+
39+
public Length(Source source, Expression src) {
40+
super(source, Arrays.asList(src));
41+
this.source = src;
42+
}
43+
44+
@Override
45+
protected TypeResolution resolveType() {
46+
if (!childrenResolved()) {
47+
return new TypeResolution("Unresolved children");
48+
}
49+
50+
return isStringAndExact(source, sourceText(), ParamOrdinal.DEFAULT);
51+
}
52+
53+
@Override
54+
protected Pipe makePipe() {
55+
return new LengthFunctionPipe(source(), this, Expressions.pipe(source));
56+
}
57+
58+
@Override
59+
public boolean foldable() {
60+
return source.foldable();
61+
}
62+
63+
@Override
64+
public Object fold() {
65+
return doProcess(source.fold());
66+
}
67+
68+
@Override
69+
protected NodeInfo<? extends Expression> info() {
70+
return NodeInfo.create(this, Length::new, source);
71+
}
72+
73+
@Override
74+
public ScriptTemplate asScript() {
75+
ScriptTemplate sourceScript = asScript(source);
76+
77+
return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s)"),
78+
"length",
79+
sourceScript.template()),
80+
paramsBuilder()
81+
.script(sourceScript.params())
82+
.build(), dataType());
83+
}
84+
85+
@Override
86+
public ScriptTemplate scriptWithField(FieldAttribute field) {
87+
return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
88+
paramsBuilder().variable(field.exactAttribute().name()).build(),
89+
dataType());
90+
}
91+
92+
@Override
93+
public DataType dataType() {
94+
return DataTypes.INTEGER;
95+
}
96+
97+
@Override
98+
public Expression replaceChildren(List<Expression> newChildren) {
99+
if (newChildren.size() != 1) {
100+
throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
101+
}
102+
103+
return new Length(source(), newChildren.get(0));
104+
}
105+
106+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
7+
8+
import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
9+
import org.elasticsearch.xpack.ql.expression.Expression;
10+
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
11+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
12+
import org.elasticsearch.xpack.ql.tree.Source;
13+
14+
import java.util.Arrays;
15+
import java.util.List;
16+
import java.util.Objects;
17+
18+
public class LengthFunctionPipe extends Pipe {
19+
20+
private final Pipe source;
21+
22+
public LengthFunctionPipe(Source source, Expression expression, Pipe src) {
23+
super(source, expression, Arrays.asList(src));
24+
this.source = src;
25+
}
26+
27+
@Override
28+
public final Pipe replaceChildren(List<Pipe> newChildren) {
29+
if (newChildren.size() != 1) {
30+
throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
31+
}
32+
return replaceChildren(newChildren.get(0));
33+
}
34+
35+
@Override
36+
public final Pipe resolveAttributes(AttributeResolver resolver) {
37+
Pipe newSource = source.resolveAttributes(resolver);
38+
if (newSource == source) {
39+
return this;
40+
}
41+
return replaceChildren(newSource);
42+
}
43+
44+
@Override
45+
public boolean supportedByAggsOnlyQuery() {
46+
return source.supportedByAggsOnlyQuery();
47+
}
48+
49+
@Override
50+
public boolean resolved() {
51+
return source.resolved();
52+
}
53+
54+
protected Pipe replaceChildren(Pipe newSource) {
55+
return new LengthFunctionPipe(source(), expression(), newSource);
56+
}
57+
58+
@Override
59+
public final void collectFields(QlSourceBuilder sourceBuilder) {
60+
source.collectFields(sourceBuilder);
61+
}
62+
63+
@Override
64+
protected NodeInfo<LengthFunctionPipe> info() {
65+
return NodeInfo.create(this, LengthFunctionPipe::new, expression(), source);
66+
}
67+
68+
@Override
69+
public LengthFunctionProcessor asProcessor() {
70+
return new LengthFunctionProcessor(source.asProcessor());
71+
}
72+
73+
public Pipe src() {
74+
return source;
75+
}
76+
77+
@Override
78+
public int hashCode() {
79+
return Objects.hash(source);
80+
}
81+
82+
@Override
83+
public boolean equals(Object obj) {
84+
if (this == obj) {
85+
return true;
86+
}
87+
88+
if (obj == null || getClass() != obj.getClass()) {
89+
return false;
90+
}
91+
92+
return Objects.equals(source, ((LengthFunctionPipe) obj).source);
93+
}
94+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
7+
8+
import org.elasticsearch.common.io.stream.StreamInput;
9+
import org.elasticsearch.common.io.stream.StreamOutput;
10+
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
11+
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
12+
13+
import java.io.IOException;
14+
import java.util.Objects;
15+
16+
public class LengthFunctionProcessor implements Processor {
17+
18+
public static final String NAME = "slen";
19+
20+
private final Processor source;
21+
22+
public LengthFunctionProcessor(Processor source) {
23+
this.source = source;
24+
}
25+
26+
public LengthFunctionProcessor(StreamInput in) throws IOException {
27+
source = in.readNamedWriteable(Processor.class);
28+
}
29+
30+
@Override
31+
public final void writeTo(StreamOutput out) throws IOException {
32+
out.writeNamedWriteable(source);
33+
}
34+
35+
@Override
36+
public Object process(Object input) {
37+
return doProcess(source.process(input));
38+
}
39+
40+
public static Object doProcess(Object source) {
41+
if (source == null) {
42+
return null;
43+
}
44+
if (source instanceof String == false && source instanceof Character == false) {
45+
throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source);
46+
}
47+
48+
return source.toString().length();
49+
}
50+
51+
protected Processor source() {
52+
return source;
53+
}
54+
55+
@Override
56+
public boolean equals(Object obj) {
57+
if (this == obj) {
58+
return true;
59+
}
60+
61+
if (obj == null || getClass() != obj.getClass()) {
62+
return false;
63+
}
64+
65+
return Objects.equals(source(), ((LengthFunctionProcessor) obj).source());
66+
}
67+
68+
@Override
69+
public int hashCode() {
70+
return Objects.hash(source());
71+
}
72+
73+
74+
@Override
75+
public String getWriteableName() {
76+
return NAME;
77+
}
78+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
88

9+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
910
import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
1011
import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
1112

@@ -18,6 +19,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
1819

1920
InternalEqlScriptUtils() {}
2021

22+
public static Integer length(String s) {
23+
return (Integer) LengthFunctionProcessor.doProcess(s);
24+
}
25+
2126
public static String substring(String s, Number start, Number end) {
2227
return (String) SubstringFunctionProcessor.doProcess(s, start, end);
2328
}

‎x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,5 +55,6 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
5555
#
5656
# ASCII Functions
5757
#
58+
Integer length(String)
5859
String substring(String, Number, Number)
5960
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
8+
9+
import org.elasticsearch.test.ESTestCase;
10+
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
11+
import org.elasticsearch.xpack.ql.expression.Literal;
12+
import org.elasticsearch.xpack.ql.expression.LiteralTests;
13+
14+
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
15+
import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
16+
import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
17+
import static org.hamcrest.Matchers.startsWith;
18+
19+
public class LengthProcessorTests extends ESTestCase {
20+
21+
public void testLengthFunctionWithValidInput() {
22+
assertEquals(9, new Length(EMPTY, l("foobarbar")).makePipe().asProcessor().process(null));
23+
assertEquals(0, new Length(EMPTY, l("")).makePipe().asProcessor().process(null));
24+
assertEquals(1, new Length(EMPTY, l('f')).makePipe().asProcessor().process(null));
25+
}
26+
27+
public void testLengthFunctionInputsValidation() {
28+
QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
29+
() -> new Length(EMPTY, l(5)).makePipe().asProcessor().process(null));
30+
assertEquals("A string/char is required; received [5]", siae.getMessage());
31+
siae = expectThrows(QlIllegalArgumentException.class, () -> new Length(EMPTY, l(true)).makePipe().asProcessor().process(null));
32+
assertEquals("A string/char is required; received [true]", siae.getMessage());
33+
}
34+
35+
public void testLengthFunctionWithRandomInvalidDataType() {
36+
Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral());
37+
QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
38+
() -> new Length(EMPTY, literal).makePipe().asProcessor().process(null));
39+
assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
40+
}
41+
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public abstract class AbstractQueryFolderTestCase extends ESTestCase {
2626
protected Optimizer optimizer = new Optimizer();
2727
protected Planner planner = new Planner();
2828

29-
protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
29+
protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json", true)));
3030

3131
protected PhysicalPlan plan(IndexResolution resolution, String eql) {
3232
return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,12 @@ public void testPropertyEquationInClauseFilterUnsupported() {
2424
assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
2525
"offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
2626
}
27+
28+
public void testLengthFunctionWithInexact() {
29+
VerificationException e = expectThrows(VerificationException.class,
30+
() -> plan("process where length(plain_text) > 0"));
31+
String msg = e.getMessage();
32+
assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
33+
+ "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
34+
}
2735
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public void test() {
9595
PhysicalPlan p = plan(query);
9696
assertEquals(EsQueryExec.class, p.getClass());
9797
EsQueryExec eqe = (EsQueryExec) p;
98-
assertEquals(25, eqe.output().size());
98+
assertEquals(27, eqe.output().size());
9999
assertEquals(KEYWORD, eqe.output().get(0).dataType());
100100

101101
final String query = eqe.queryContainer().toString().replaceAll("\\s+", "");

0 commit comments

Comments
 (0)