diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java index 5c768a7dcb1ef..690483bc8d81d 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -66,7 +66,7 @@ public void setup() throws IOException { ); includesSet = Set.of(fetchContext.includes()); excludesSet = Set.of(fetchContext.excludes()); - parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet); + parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false); } private BytesReference read300BytesExample() throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java index 6f4e926dbe969..02b296fecb4b8 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java @@ -61,6 +61,7 @@ public class FilterContentBenchmark { private BytesReference source; private XContentParserConfiguration parserConfig; private Set filters; + private XContentParserConfiguration parserConfigMatchDotsInFieldNames; @Setup public void setup() throws IOException { @@ -72,7 +73,8 @@ public void setup() throws IOException { }; source = readSource(sourceFile); filters = buildFilters(); - parserConfig = buildParseConfig(); + parserConfig = buildParseConfig(false); + parserConfigMatchDotsInFieldNames = buildParseConfig(true); } private Set buildFilters() { @@ -105,9 +107,14 @@ public BytesReference filterWithParserConfigCreated() throws IOException { return filter(this.parserConfig); } + @Benchmark + public BytesReference filterWithParserConfigCreatedMatchDotsInFieldNames() throws IOException { + return filter(this.parserConfigMatchDotsInFieldNames); + } + @Benchmark public BytesReference filterWithNewParserConfig() throws IOException { - XContentParserConfiguration contentParserConfiguration = buildParseConfig(); + XContentParserConfiguration contentParserConfiguration = buildParseConfig(false); return filter(contentParserConfiguration); } @@ -152,7 +159,7 @@ public BytesReference filterWithBuilder() throws IOException { } } - private XContentParserConfiguration buildParseConfig() { + private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNames) { Set includes; Set excludes; if (inclusive) { @@ -162,7 +169,7 @@ private XContentParserConfiguration buildParseConfig() { includes = null; excludes = filters; } - return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes); + return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames); } private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java index 21765a7e3a54d..66b4a62a7095d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java @@ -32,7 +32,8 @@ public class XContentParserConfiguration { DeprecationHandler.THROW_UNSUPPORTED_OPERATION, RestApiVersion.current(), null, - null + null, + false ); final NamedXContentRegistry registry; @@ -40,26 +41,36 @@ public class XContentParserConfiguration { final RestApiVersion restApiVersion; final FilterPath[] includes; final FilterPath[] excludes; + final boolean filtersMatchFieldNamesWithDots; private XContentParserConfiguration( NamedXContentRegistry registry, DeprecationHandler deprecationHandler, RestApiVersion restApiVersion, FilterPath[] includes, - FilterPath[] excludes + FilterPath[] excludes, + boolean filtersMatchFieldNamesWithDots ) { this.registry = registry; this.deprecationHandler = deprecationHandler; this.restApiVersion = restApiVersion; this.includes = includes; this.excludes = excludes; + this.filtersMatchFieldNamesWithDots = filtersMatchFieldNamesWithDots; } /** * Replace the registry backing {@link XContentParser#namedObject}. */ public XContentParserConfiguration withRegistry(NamedXContentRegistry registry) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public NamedXContentRegistry registry() { @@ -71,7 +82,14 @@ public NamedXContentRegistry registry() { * a deprecated field. */ public XContentParserConfiguration withDeprecationHandler(DeprecationHandler deprecationHandler) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public DeprecationHandler deprecationHandler() { @@ -83,7 +101,14 @@ public DeprecationHandler deprecationHandler() { * {@link RestApiVersion}. */ public XContentParserConfiguration withRestApiVersion(RestApiVersion restApiVersion) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public RestApiVersion restApiVersion() { @@ -93,13 +118,18 @@ public RestApiVersion restApiVersion() { /** * Replace the configured filtering. */ - public XContentParserConfiguration withFiltering(Set includeStrings, Set excludeStrings) { + public XContentParserConfiguration withFiltering( + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ) { return new XContentParserConfiguration( registry, deprecationHandler, restApiVersion, FilterPath.compile(includeStrings), - FilterPath.compile(excludeStrings) + FilterPath.compile(excludeStrings), + filtersMatchFieldNamesWithDots ); } @@ -112,10 +142,20 @@ public JsonParser filter(JsonParser parser) { throw new UnsupportedOperationException("double wildcards are not supported in filtered excludes"); } } - filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(excludes, false), true, true); + filtered = new FilteringParserDelegate( + filtered, + new FilterPathBasedFilter(excludes, false, filtersMatchFieldNamesWithDots), + true, + true + ); } if (includes != null) { - filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(includes, true), true, true); + filtered = new FilteringParserDelegate( + filtered, + new FilterPathBasedFilter(includes, true, filtersMatchFieldNamesWithDots), + true, + true + ); } return filtered; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java index 9bb289bfccd47..350611ef01a85 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java @@ -68,13 +68,22 @@ private boolean isFinalNode() { * if current node is a double wildcard node, the node will also add to nextFilters. * @param name the xcontent property name * @param nextFilters nextFilters is a List, used to check the inner property of name + * @param matchFieldNamesWithDots support dot in field name or not * @return true if the name equal a final node, otherwise return false */ - boolean matches(String name, List nextFilters) { + boolean matches(String name, List nextFilters, boolean matchFieldNamesWithDots) { if (nextFilters == null) { return false; } + // match dot first + if (matchFieldNamesWithDots) { + // contains dot and not the first or last char + int dotIndex = name.indexOf('.'); + if ((dotIndex != -1) && (dotIndex != 0) && (dotIndex != name.length() - 1)) { + return matchFieldNamesWithDots(name, dotIndex, nextFilters); + } + } FilterPath termNode = termsChildren.get(name); if (termNode != null) { if (termNode.isFinalNode()) { @@ -102,6 +111,25 @@ boolean matches(String name, List nextFilters) { return false; } + private boolean matchFieldNamesWithDots(String name, int dotIndex, List nextFilters) { + String prefixName = name.substring(0, dotIndex); + String suffixName = name.substring(dotIndex + 1); + List prefixFilterPath = new ArrayList<>(); + boolean prefixMatch = matches(prefixName, prefixFilterPath, true); + // if prefixMatch return true(because prefix is a final FilterPath node) + if (prefixMatch) { + return true; + } + // if has prefixNextFilter, use them to match suffix + for (FilterPath filter : prefixFilterPath) { + boolean matches = filter.matches(suffixName, nextFilters, true); + if (matches) { + return true; + } + } + return false; + } + private static class FilterPathBuilder { private class BuildNode { private final Map children; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java index dc0ca01cc3f3c..ef9411ba2f79f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java @@ -42,16 +42,19 @@ public String toString() { private final boolean inclusive; - public FilterPathBasedFilter(FilterPath[] filters, boolean inclusive) { + private final boolean matchFieldNamesWithDots; + + public FilterPathBasedFilter(FilterPath[] filters, boolean inclusive, boolean matchFieldNamesWithDots) { if (filters == null || filters.length == 0) { throw new IllegalArgumentException("filters cannot be null or empty"); } this.inclusive = inclusive; this.filters = filters; + this.matchFieldNamesWithDots = matchFieldNamesWithDots; } public FilterPathBasedFilter(Set filters, boolean inclusive) { - this(FilterPath.compile(filters), inclusive); + this(FilterPath.compile(filters), inclusive, false); } /** @@ -61,14 +64,18 @@ private TokenFilter evaluate(String name, FilterPath[] filterPaths) { if (filterPaths != null) { List nextFilters = new ArrayList<>(); for (FilterPath filter : filterPaths) { - boolean matches = filter.matches(name, nextFilters); + boolean matches = filter.matches(name, nextFilters, matchFieldNamesWithDots); if (matches) { return MATCHING; } } if (nextFilters.isEmpty() == false) { - return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[nextFilters.size()]), inclusive); + return new FilterPathBasedFilter( + nextFilters.toArray(new FilterPath[nextFilters.size()]), + inclusive, + matchFieldNamesWithDots + ); } } return NO_MATCHING; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 3db8a13992ef4..5d535b2446358 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -21,17 +21,316 @@ import java.io.IOException; import java.util.Arrays; import java.util.Set; +import java.util.stream.IntStream; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; +import static java.util.stream.Collectors.joining; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public abstract class AbstractXContentFilteringTestCase extends AbstractFilteringTestCase { + public void testSingleFieldObject() throws IOException { + Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); + Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); + testFilter(expected, sample, singleton("foo.bar"), emptySet()); + testFilter(expected, sample, emptySet(), singleton("foo.baz")); + testFilter(expected, sample, singleton("foo"), singleton("foo.baz")); + + expected = builder -> builder.startObject().endObject(); + testFilter(expected, sample, emptySet(), singleton("foo.bar")); + testFilter(expected, sample, singleton("foo"), singleton("foo.b*")); + } + + public void testDotInIncludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("foo.bar"), + emptySet(), + false + ); + } + + public void testDotInIncludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotInExcludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton("foo.bar"), + false + ); + } + + public void testDotInExcludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotInIncludedObjectNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton("foo.bar"), + emptySet(), + false + ); + } + + public void testDotInIncludedObjectNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotInExcludedObjectNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("foo.bar"), + false + ); + } + + public void testDotInExcludedObjectNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotInIncludedFieldNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + emptySet(), + false + ); + } + + public void testDotInIncludedFieldNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + emptySet(), + true + ); + } + + public void testDotInExcludedFieldNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton(randomFrom("foo.*", "foo.*ar")), + false + ); + } + + public void testDotInExcludedFieldNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + true + ); + } + + public void testDotInIncludedObjectNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton(randomFrom("foo.*", "f*.bar", "foo.*ar")), + emptySet(), + false + ); + } + + public void testDotInIncludedObjectNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + emptySet(), + true + ); + } + + public void testDotInExcludedObjectNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + false + ); + } + public void testDotInExcludedObjectNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + true + ); + } + + public void testDotInStarMatchDotsInNamesUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("f*r"), + emptySet(), + false + ); + } + + public void testDotInStarMatchDotsInNamesConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("f*r"), + emptySet(), + true + ); + } + + public void testTwoDotsInIncludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar.baz"), + emptySet(), + false + ); + } + + public void testTwoDotsInIncludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar.baz"), + emptySet(), + true + ); + } + + public void testManyDotsInIncludedFieldName() throws IOException { + String name = IntStream.rangeClosed(1, 100).mapToObj(i -> "a").collect(joining(".")); + testFilter( + builder -> builder.startObject().field(name, "test").endObject(), + builder -> builder.startObject().field(name, "test").endObject(), + singleton(name), + emptySet(), + true + ); + } + + public void testDotsInIncludedFieldNamePrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotsInExcludedFieldNamePrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotsInIncludedFieldNamePatternPrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("f*.*r"), + emptySet(), + true + ); + } + + public void testDotsInExcludedFieldNamePatternPrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("f*.*r"), + true + ); + } + + public void testDotsAndDoubleWildcardInIncludedFieldName() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("**.baz"), + emptySet(), + true + ); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/80160") + public void testDotsAndDoubleWildcardInExcludedFieldName() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("**.baz"), + true + ); + // bug of double wildcard in excludes report in https://github.com/FasterXML/jackson-core/issues/700 + testFilter( + builder -> builder.startObject().startObject("foo").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo").field("bar", "test").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("**.bar"), + true + ); + } + + @Override protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { - assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes)); + testFilter(expected, sample, includes, excludes, false); + } + + private void testFilter(Builder expected, Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { + assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots)); } protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual); @@ -47,8 +346,9 @@ private XContentBuilder createBuilder() throws IOException { return XContentBuilder.builder(getXContentType().xContent()); } - private XContentBuilder filter(Builder sample, Set includes, Set excludes) throws IOException { - if (randomBoolean()) { + private XContentBuilder filter(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { + if (matchFieldNamesWithDots == false && randomBoolean()) { return filterOnBuilder(sample, includes, excludes); } FilterPath[] excludesFilter = FilterPath.compile(excludes); @@ -58,21 +358,26 @@ private XContentBuilder filter(Builder sample, Set includes, Set * filtering produced weird invalid json. Just field names * and no objects?! Weird. Anyway, we can't use it. */ + assertFalse("can't filter on builder with dotted wildcards in exclude", matchFieldNamesWithDots); return filterOnBuilder(sample, includes, excludes); } - return filterOnParser(sample, includes, excludes); + return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots); } private XContentBuilder filterOnBuilder(Builder sample, Set includes, Set excludes) throws IOException { return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes)); } - private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes) throws IOException { + private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { try (XContentBuilder builtSample = sample.apply(createBuilder())) { BytesReference sampleBytes = BytesReference.bytes(builtSample); try ( XContentParser parser = getXContentType().xContent() - .createParser(XContentParserConfiguration.EMPTY.withFiltering(includes, excludes), sampleBytes.streamInput()) + .createParser( + XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots), + sampleBytes.streamInput() + ) ) { XContentBuilder result = createBuilder(); if (sampleBytes.get(sampleBytes.length() - 1) == '\n') { @@ -87,19 +392,6 @@ private XContentBuilder filterOnParser(Builder sample, Set includes, Set } } - public void testSingleFieldObject() throws IOException { - final Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); - - Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); - testFilter(expected, sample, singleton("foo.bar"), emptySet()); - testFilter(expected, sample, emptySet(), singleton("foo.baz")); - testFilter(expected, sample, singleton("foo"), singleton("foo.baz")); - - expected = builder -> builder.startObject().endObject(); - testFilter(expected, sample, emptySet(), singleton("foo.bar")); - testFilter(expected, sample, singleton("foo"), singleton("foo.b*")); - } - static void assertXContentBuilderAsString(final XContentBuilder expected, final XContentBuilder actual) { assertThat(Strings.toString(actual), is(Strings.toString(expected))); } diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java index 8545a497fdb68..465bab256ac3b 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java @@ -32,7 +32,7 @@ public void testSimpleFilterPath() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; - assertThat(filterPath.matches("test", nextFilters), is(true)); + assertThat(filterPath.matches("test", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -46,12 +46,12 @@ public void testFilterPathWithSubField() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 1); } @@ -65,18 +65,18 @@ public void testFilterPathWithSubFields() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(false)); + assertThat(filterPath.matches("bar", nextFilters, false), is(false)); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("quz", nextFilters), is(true)); + assertThat(filterPath.matches("quz", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -102,25 +102,25 @@ public void testFilterPathWithEscapedDots() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("w", nextFilters), is(false)); + assertThat(filterPath.matches("w", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0", nextFilters), is(false)); + assertThat(filterPath.matches("0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0", nextFilters), is(false)); + assertThat(filterPath.matches("0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("t", nextFilters), is(true)); + assertThat(filterPath.matches("t", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); input = "w\\.0\\.0\\.t"; @@ -131,7 +131,7 @@ public void testFilterPathWithEscapedDots() { nextFilters = new ArrayList<>(); filterPath = filterPaths[0]; - assertTrue(filterPath.matches("w.0.0.t", nextFilters)); + assertTrue(filterPath.matches("w.0.0.t", nextFilters, false)); assertEquals(nextFilters.size(), 0); input = "w\\.0.0\\.t"; @@ -143,13 +143,13 @@ public void testFilterPathWithEscapedDots() { filterPath = filterPaths[0]; nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("w.0", nextFilters), is(false)); + assertThat(filterPath.matches("w.0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0.t", nextFilters), is(true)); + assertThat(filterPath.matches("0.t", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -161,7 +161,7 @@ public void testSimpleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(true)); + assertThat(filterPath.matches("foo", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -175,19 +175,19 @@ public void testWildcardInNameFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); - assertThat(filterPath.matches("flo", nextFilters), is(false)); + assertThat(filterPath.matches("flo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 2); - assertThat(filterPath.matches("foooo", nextFilters), is(false)); + assertThat(filterPath.matches("foooo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 3); - assertThat(filterPath.matches("boo", nextFilters), is(false)); + assertThat(filterPath.matches("boo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 3); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -199,7 +199,7 @@ public void testDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(true)); + assertThat(filterPath.matches("foo", nextFilters, false), is(true)); assertThat(filterPath.hasDoubleWildcard(), is(true)); assertEquals(nextFilters.size(), 0); } @@ -214,13 +214,13 @@ public void testStartsWithDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -234,19 +234,19 @@ public void testContainsDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -261,38 +261,38 @@ public void testMultipleFilterPaths() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(false)); + assertThat(filterPath.matches("bar", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 2); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test2", nextFilters), is(true)); + assertThat(filterPath.matches("test2", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); // test.dot\.ted filterPath = filterPaths[0]; nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("dot.ted", nextFilters), is(true)); + assertThat(filterPath.matches("dot.ted", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -335,7 +335,71 @@ public void testNoMatchesFilter() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; - assertFalse(filterPath.matches(randomAlphaOfLength(10), nextFilters)); + assertFalse(filterPath.matches(randomAlphaOfLength(10), nextFilters, false)); + assertEquals(nextFilters.size(), 0); + } + + public void testDotInFieldName() { + // FilterPath match + FilterPath[] filterPaths = FilterPath.compile(singleton("foo")); + List nextFilters = new ArrayList<>(); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath not match + filterPaths = FilterPath.compile(singleton("bar")); + assertFalse(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath equals to fieldName + filterPaths = FilterPath.compile(singleton("foo.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath longer than fieldName + filterPaths = FilterPath.compile(singleton("foo.bar.test")); + assertFalse(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + + // partial match + filterPaths = FilterPath.compile(singleton("foo.bar.test")); + assertFalse(filterPaths[0].matches("foo.bar.text", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // wildcard + filterPaths = FilterPath.compile(singleton("*.bar")); + assertFalse(filterPaths[0].matches("foo", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + filterPaths = FilterPath.compile(singleton("*.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("f*.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("foo.*")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("foo.*ar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("*.*")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // test double wildcard + filterPaths = FilterPath.compile(singleton("**.c")); + assertFalse(filterPaths[0].matches("a.b", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + assertTrue(filterPaths[0].matches("a.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.d.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.c.d", nextFilters, true)); assertEquals(nextFilters.size(), 0); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index cb9489b7745c1..6cc399f598155 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -404,7 +404,8 @@ class DataStream implements IndexAbstraction { public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering( Set.of("@timestamp"), - null + null, + false ); private final org.elasticsearch.cluster.metadata.DataStream dataStream; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 544933c29fac9..64a2a2b093468 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -214,7 +214,7 @@ private static class ExtractFromSource extends IndexRouting { if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(metadata.getRoutingPaths()), null); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(metadata.getRoutingPaths()), null, true); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 0c02f9770b72f..d120983ce562c 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -232,7 +232,9 @@ public static Map convertToMap( @Nullable Set include, @Nullable Set exclude ) throws ElasticsearchParseException { - try (XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude), input)) { + try ( + XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input) + ) { return ordered ? parser.mapOrdered() : parser.map(); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); @@ -266,7 +268,7 @@ public static Map convertToMap( ) throws ElasticsearchParseException { try ( XContentParser parser = xContent.createParser( - XContentParserConfiguration.EMPTY.withFiltering(include, exclude), + XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), bytes, offset, length