diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle new file mode 100644 index 0000000000000..adad51f4f9ceb --- /dev/null +++ b/libs/web-utils/build.gradle @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +dependencies { + api project(':libs:core') + implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" + implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" + + testImplementation(project(":test:framework")) +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses( + 'org.apache.commons.codec.binary.Base64', + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + ) +} diff --git a/modules/ingest-common/licenses/httpclient-LICENSE.txt b/libs/web-utils/licenses/httpclient-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/httpclient-LICENSE.txt rename to libs/web-utils/licenses/httpclient-LICENSE.txt diff --git a/modules/ingest-common/licenses/httpclient-NOTICE.txt b/libs/web-utils/licenses/httpclient-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/httpclient-NOTICE.txt rename to libs/web-utils/licenses/httpclient-NOTICE.txt diff --git a/modules/ingest-common/licenses/httpcore-LICENSE.txt b/libs/web-utils/licenses/httpcore-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/httpcore-LICENSE.txt rename to libs/web-utils/licenses/httpcore-LICENSE.txt diff --git a/modules/ingest-common/licenses/httpcore-NOTICE.txt b/libs/web-utils/licenses/httpcore-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/httpcore-NOTICE.txt rename to libs/web-utils/licenses/httpcore-NOTICE.txt diff --git a/libs/web-utils/src/main/java/module-info.java b/libs/web-utils/src/main/java/module-info.java new file mode 100644 index 0000000000000..319f627d093c2 --- /dev/null +++ b/libs/web-utils/src/main/java/module-info.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.web { + requires org.elasticsearch.base; + requires org.apache.httpcomponents.httpclient; + requires org.apache.httpcomponents.httpcore; + + exports org.elasticsearch.web; +} diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java new file mode 100644 index 0000000000000..70975895428d4 --- /dev/null +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.elasticsearch.core.Nullable; + +import java.util.LinkedHashMap; + +/** + * Utility class for parsing fully qualified domain names (FQDNs) into their constituent parts: + * domain, registered domain, top-level domain (eTLD), and subdomain. + *

+ * This class uses the public suffix list to accurately determine domain boundaries. + * For example, given "www.example.co.uk": + *

+ * + * @see Public Suffix List + * @see eTLD (effective Top-Level Domain) + */ +public class RegisteredDomain { + + public static final String DOMAIN = "domain"; + public static final String REGISTERED_DOMAIN = "registered_domain"; + public static final String eTLD = "top_level_domain"; + public static final String SUBDOMAIN = "subdomain"; + + public static final LinkedHashMap> REGISTERED_DOMAIN_INFO_FIELDS; + + static { + REGISTERED_DOMAIN_INFO_FIELDS = new LinkedHashMap<>(); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(DOMAIN, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(REGISTERED_DOMAIN, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(eTLD, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(SUBDOMAIN, String.class); + } + + private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); + + public static boolean parseRegisteredDomainInfo(@Nullable final String fqdn, final RegisteredDomainInfoCollector collector) { + if (fqdn == null || fqdn.isBlank()) { + return false; + } + String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); + if (registeredDomain == null) { + if (SUFFIX_MATCHER.matches(fqdn)) { + collector.topLevelDomain(fqdn); + collector.domain(fqdn); + return true; + } + return false; + } + int indexOfDot = registeredDomain.indexOf('.') + 1; + if (indexOfDot > 0 && indexOfDot < registeredDomain.length()) { + collector.domain(fqdn); + collector.registeredDomain(registeredDomain); + collector.topLevelDomain(registeredDomain.substring(indexOfDot)); + int subdomainIndex = fqdn.lastIndexOf("." + registeredDomain); + if (subdomainIndex > 0) { + collector.subdomain(fqdn.substring(0, subdomainIndex)); + } + return true; + } + return false; + } + + public static LinkedHashMap> getRegisteredDomainInfoFields() { + return REGISTERED_DOMAIN_INFO_FIELDS; + } + + /** + * A collector for registered domain information. + * Implementation can be specific to the use case, for example - it can write parsed info directly to the collecting data structure. + */ + public interface RegisteredDomainInfoCollector { + /** + * @param domain the domain name + */ + void domain(String domain); + + /** + * @param registeredDomain the registered domain name + */ + void registeredDomain(String registeredDomain); + + /** + * @param topLevelDomain the top level domain, n.b. eTLD + */ + void topLevelDomain(String topLevelDomain); + + /** + * @param subdomain the subdomain name + */ + void subdomain(String subdomain); + } + + public record DomainInfo(String domain, String registeredDomain, String eTLD, String subdomain) { + static class Factory implements RegisteredDomainInfoCollector { + String domain; + String registeredDomain; + String topLevelDomain; + String subdomain; + + @Override + public void domain(String domain) { + this.domain = domain; + } + + @Override + public void registeredDomain(String registeredDomain) { + this.registeredDomain = registeredDomain; + } + + @Override + public void topLevelDomain(String topLevelDomain) { + this.topLevelDomain = topLevelDomain; + } + + @Override + public void subdomain(String subdomain) { + this.subdomain = subdomain; + } + + public DomainInfo build() { + return new DomainInfo(domain, registeredDomain, topLevelDomain, subdomain); + } + } + } + + public static DomainInfo parseRegisteredDomainInfo(@Nullable final String fqdn) { + DomainInfo.Factory factory = new DomainInfo.Factory(); + boolean infoFound = parseRegisteredDomainInfo(fqdn, factory); + return infoFound ? factory.build() : null; + } +} diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java new file mode 100644 index 0000000000000..14ef805348536 --- /dev/null +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.core.SuppressForbidden; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +public class UriParts { + + private static final String DOMAIN = "domain"; + private static final String FRAGMENT = "fragment"; + private static final String PATH = "path"; + private static final String PORT = "port"; + private static final String QUERY = "query"; + private static final String SCHEME = "scheme"; + private static final String USER_INFO = "user_info"; + private static final String EXTENSION = "extension"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + + private static final LinkedHashMap> URI_PARTS_TYPES; + + static { + URI_PARTS_TYPES = new LinkedHashMap<>(); + URI_PARTS_TYPES.putLast(DOMAIN, String.class); + URI_PARTS_TYPES.putLast(FRAGMENT, String.class); + URI_PARTS_TYPES.putLast(PATH, String.class); + URI_PARTS_TYPES.putLast(PORT, Integer.class); + URI_PARTS_TYPES.putLast(QUERY, String.class); + URI_PARTS_TYPES.putLast(SCHEME, String.class); + URI_PARTS_TYPES.putLast(EXTENSION, String.class); + URI_PARTS_TYPES.putLast(USER_INFO, String.class); + URI_PARTS_TYPES.putLast(USERNAME, String.class); + URI_PARTS_TYPES.putLast(PASSWORD, String.class); + } + + public static LinkedHashMap> getUriPartsTypes() { + return URI_PARTS_TYPES; + } + + public static Map parse(String uriString) { + final var uriParts = new UriPartsMapCollector(); + parse(uriString, uriParts); + return uriParts; + } + + @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") + public static void parse(final String uriString, final UriPartsCollector uriPartsCollector) { + URI uri = null; + URL fallbackUrl = null; + try { + uri = new URI(uriString); + } catch (URISyntaxException e) { + try { + // noinspection deprecation + fallbackUrl = new URL(uriString); + } catch (MalformedURLException e2) { + throw new IllegalArgumentException("unable to parse URI [" + uriString + "]"); + } + } + + String domain; + String fragment; + String path; + int port; + String query; + String scheme; + String userInfo; + + if (uri != null) { + domain = uri.getHost(); + fragment = uri.getFragment(); + path = uri.getPath(); + port = uri.getPort(); + query = uri.getQuery(); + scheme = uri.getScheme(); + userInfo = uri.getUserInfo(); + } else if (fallbackUrl != null) { + domain = fallbackUrl.getHost(); + fragment = fallbackUrl.getRef(); + path = fallbackUrl.getPath(); + port = fallbackUrl.getPort(); + query = fallbackUrl.getQuery(); + scheme = fallbackUrl.getProtocol(); + userInfo = fallbackUrl.getUserInfo(); + } else { + // should never occur during processor execution + throw new IllegalArgumentException("at least one argument must be non-null"); + } + + uriPartsCollector.domain(domain); + if (fragment != null) { + uriPartsCollector.fragment(fragment); + } + if (path != null) { + uriPartsCollector.path(path); + // To avoid any issues with extracting the extension from a path that contains a dot, we explicitly extract the extension + // from the last segment in the path. + var lastSegmentIndex = path.lastIndexOf('/'); + if (lastSegmentIndex >= 0) { + var lastSegment = path.substring(lastSegmentIndex); + int periodIndex = lastSegment.lastIndexOf('.'); + if (periodIndex >= 0) { + // Don't include the dot in the extension field. + uriPartsCollector.extension(lastSegment.substring(periodIndex + 1)); + } + } + } + if (port != -1) { + uriPartsCollector.port(port); + } + if (query != null) { + uriPartsCollector.query(query); + } + uriPartsCollector.scheme(scheme); + if (userInfo != null) { + uriPartsCollector.userInfo(userInfo); + if (userInfo.contains(":")) { + int colonIndex = userInfo.indexOf(':'); + uriPartsCollector.username(userInfo.substring(0, colonIndex)); + uriPartsCollector.password(colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); + } + } + } + + /** + * A dedicated collector for URI parts. Implementation can be specific to the use case, for example, it can avoid map instance + * allocation and primitive value boxing by writing directly to the collecting data structure. + */ + public interface UriPartsCollector { + void domain(String domain); + + void fragment(String fragment); + + void path(String path); + + void extension(String extension); + + void port(int port); + + void query(String query); + + void scheme(String scheme); + + void userInfo(String userInfo); + + void username(String username); + + void password(String password); + } + + /** + * A default implementation of {@link UriPartsCollector} that writes to a {@link Map}. + */ + public static final class UriPartsMapCollector extends HashMap implements UriPartsCollector { + @Override + public void domain(String domain) { + if (domain != null) { + put(DOMAIN, domain); + } + } + + @Override + public void fragment(String fragment) { + if (fragment != null) { + put(FRAGMENT, fragment); + } + } + + @Override + public void path(String path) { + if (path != null) { + put(PATH, path); + } + } + + @Override + public void extension(String extension) { + if (extension != null) { + put(EXTENSION, extension); + } + } + + @Override + public void port(int port) { + if (port >= 0) { + put(PORT, port); + } + } + + @Override + public void query(String query) { + if (query != null) { + put(QUERY, query); + } + } + + @Override + public void scheme(String scheme) { + if (scheme != null) { + put(SCHEME, scheme); + } + } + + @Override + public void userInfo(String userInfo) { + if (userInfo != null) { + put(USER_INFO, userInfo); + } + } + + @Override + public void username(String username) { + if (username != null) { + put(USERNAME, username); + } + } + + @Override + public void password(String password) { + if (password != null) { + put(PASSWORD, password); + } + } + } +} diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java new file mode 100644 index 0000000000000..2016bfd6da2cd --- /dev/null +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.web.RegisteredDomain.parseRegisteredDomainInfo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +@ESTestCase.WithoutEntitlements +public class RegisteredDomainTests extends ESTestCase { + + public void testGetRegisteredDomain() { + assertThat( + parseRegisteredDomainInfo("www.google.com"), + is(new RegisteredDomain.DomainInfo("www.google.com", "google.com", "com", "www")) + ); + assertThat(parseRegisteredDomainInfo("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); + assertThat(parseRegisteredDomainInfo(null), nullValue()); + assertThat(parseRegisteredDomainInfo(""), nullValue()); + assertThat(parseRegisteredDomainInfo(" "), nullValue()); + assertThat(parseRegisteredDomainInfo("."), nullValue()); + assertThat(parseRegisteredDomainInfo("$"), nullValue()); + assertThat(parseRegisteredDomainInfo("foo.bar.baz"), nullValue()); + assertThat( + parseRegisteredDomainInfo("www.books.amazon.co.uk"), + is(new RegisteredDomain.DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) + ); + // Verify "com" is returned as the eTLD, for that FQDN or subdomain + assertThat(parseRegisteredDomainInfo("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); + assertThat( + parseRegisteredDomainInfo("example.com"), + is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null)) + ); + assertThat( + parseRegisteredDomainInfo("googleapis.com"), + is(new RegisteredDomain.DomainInfo("googleapis.com", "googleapis.com", "com", null)) + ); + assertThat( + parseRegisteredDomainInfo("content-autofill.googleapis.com"), + is(new RegisteredDomain.DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) + ); + // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain + assertThat( + parseRegisteredDomainInfo("global.ssl.fastly.net"), + is(new RegisteredDomain.DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) + ); + assertThat( + parseRegisteredDomainInfo("1.www.global.ssl.fastly.net"), + is(new RegisteredDomain.DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) + ); + } +} diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java new file mode 100644 index 0000000000000..8a8c4c5701e77 --- /dev/null +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java @@ -0,0 +1,233 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; + +public class UriPartsTests extends ESTestCase { + + public void testUriParts() { + + // simple URI + testUriParsing("http://www.google.com", Map.of("scheme", "http", "domain", "www.google.com", "path", "")); + + // custom port + testUriParsing("http://www.google.com:88", Map.of("scheme", "http", "domain", "www.google.com", "path", "", "port", 88)); + + // file + testUriParsing( + "http://www.google.com:88/google.png", + Map.of("scheme", "http", "domain", "www.google.com", "extension", "png", "path", "/google.png", "port", 88) + ); + + // fragment + testUriParsing( + "https://www.google.com:88/foo#bar", + Map.of("scheme", "https", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + + // path, extension + testUriParsing( + "https://www.google.com:88/foo.jpg", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo.jpg", "extension", "jpg", "port", 88) + ); + + // query + testUriParsing( + "https://www.google.com:88/foo?key=val", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo", "query", "key=val", "port", 88) + ); + + // user_info + testUriParsing( + "https://user:pw@www.google.com:88/foo", + Map.of( + "scheme", + "https", + "domain", + "www.google.com", + "path", + "/foo", + "port", + 88, + "user_info", + "user:pw", + "username", + "user", + "password", + "pw" + ) + ); + + // user_info without password + testUriParsing( + "https://user:@www.google.com:88/foo", + Map.of( + "scheme", + "https", + "domain", + "www.google.com", + "path", + "/foo", + "port", + 88, + "user_info", + "user:", + "username", + "user", + "password", + "" + ) + ); + + // everything! + testUriParsing( + "https://user:pw@testing.google.com:8080/foo/bar?foo1=bar1&foo2=bar2#anchorVal", + Map.of( + "scheme", + "https", + "domain", + "testing.google.com", + "fragment", + "anchorVal", + "path", + "/foo/bar", + "port", + 8080, + "username", + "user", + "password", + "pw", + "user_info", + "user:pw", + "query", + "foo1=bar1&foo2=bar2" + ) + ); + + // non-http schemes + testUriParsing( + "ftp://ftp.is.co.za/rfc/rfc1808.txt", + Map.of("scheme", "ftp", "path", "/rfc/rfc1808.txt", "extension", "txt", "domain", "ftp.is.co.za") + ); + + testUriParsing("telnet://192.0.2.16:80/", Map.of("scheme", "telnet", "path", "/", "port", 80, "domain", "192.0.2.16")); + + testUriParsing( + "ldap://[2001:db8::7]/c=GB?objectClass?one", + Map.of("scheme", "ldap", "path", "/c=GB", "query", "objectClass?one", "domain", "[2001:db8::7]") + ); + + // keep original + testUriParsing( + "http://www.google.com:88/foo#bar", + Map.of("scheme", "http", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + + // remove if successful + testUriParsing( + "http://www.google.com:88/foo#bar", + Map.of("scheme", "http", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + } + + public void testUrlWithCharactersNotToleratedByUri() throws Exception { + testUriParsing( + "http://www.google.com/path with spaces", + Map.of("scheme", "http", "domain", "www.google.com", "path", "/path with spaces") + ); + + testUriParsing( + "https://user:pw@testing.google.com:8080/foo with space/bar?foo1=bar1&foo2=bar2#anchorVal", + Map.of( + "scheme", + "https", + "domain", + "testing.google.com", + "fragment", + "anchorVal", + "path", + "/foo with space/bar", + "port", + 8080, + "username", + "user", + "password", + "pw", + "user_info", + "user:pw", + "query", + "foo1=bar1&foo2=bar2" + ) + ); + } + + public void testDotPathWithoutExtension() throws Exception { + testUriParsing( + "https://www.google.com/path.withdot/filenamewithoutextension", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/path.withdot/filenamewithoutextension") + ); + } + + public void testDotPathWithExtension() throws Exception { + testUriParsing( + "https://www.google.com/path.withdot/filenamewithextension.txt", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/path.withdot/filenamewithextension.txt", "extension", "txt") + ); + } + + /** + * This test verifies that we return an empty extension instead of null if the URI ends with a period. This is probably + * not behaviour we necessarily want to keep forever, but this test ensures that we're conscious about changing that behaviour. + */ + public void testEmptyExtension() throws Exception { + testUriParsing( + "https://www.google.com/foo/bar.", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo/bar.", "extension", "") + ); + } + + public void testInvalidUri() { + final String uri = "not:\\/_a_valid_uri"; + expectThrows(IllegalArgumentException.class, containsString("unable to parse URI [" + uri + "]"), () -> UriParts.parse(uri)); + } + + public void testNullValue() { + expectThrows(NullPointerException.class, () -> UriParts.parse(null)); + } + + private void testUriParsing(String uri, Map expectedValues) { + + Map actualValues = UriParts.parse(uri); + Map> expectedTypes = UriParts.getUriPartsTypes(); + + for (Map.Entry entry : expectedValues.entrySet()) { + String partName = entry.getKey(); + assertThat(actualValues, hasEntry(partName, entry.getValue())); + + Class expectedType = expectedTypes.get(partName); + assertNotNull("No type defined for key '" + partName + "'", expectedType); + assertTrue("Type mismatch for key '" + partName + "' for URI: " + uri, expectedType.isInstance(actualValues.get(partName))); + } + + // ensure that every key returned by parse() has a corresponding type definition in getUriPartsTypes() + for (Map.Entry entry : actualValues.entrySet()) { + assertTrue( + "Key '" + entry.getKey() + "' from parsed URI '" + uri + "' is not defined in UriParts.getUriPartsTypes()", + expectedTypes.containsKey(entry.getKey()) + ); + } + } +} diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 74d753af69325..5df689a88b93e 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -22,8 +22,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') api project(':libs:grok') api project(':libs:dissect') - implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" - implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" + api project(':libs:web-utils') } restResources { diff --git a/modules/ingest-common/src/main/java/module-info.java b/modules/ingest-common/src/main/java/module-info.java index c3b3ab90892d9..2972996613b0d 100644 --- a/modules/ingest-common/src/main/java/module-info.java +++ b/modules/ingest-common/src/main/java/module-info.java @@ -14,8 +14,8 @@ requires org.elasticsearch.painless.spi; requires org.elasticsearch.server; requires org.elasticsearch.xcontent; + requires org.elasticsearch.web; - requires org.apache.httpcomponents.httpclient; requires org.apache.logging.log4j; requires org.apache.lucene.analysis.common; requires org.jruby.joni; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index fa6df0c7b4f78..f67b09955291a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -9,22 +9,18 @@ package org.elasticsearch.ingest.common; -import org.apache.http.conn.util.PublicSuffixMatcher; -import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.web.RegisteredDomain; import java.util.Map; public class RegisteredDomainProcessor extends AbstractProcessor { public static final String TYPE = "registered_domain"; - private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); private final String field; private final String targetField; @@ -52,86 +48,25 @@ public boolean getIgnoreMissing() { @Override public IngestDocument execute(IngestDocument document) throws Exception { final String fqdn = document.getFieldValue(field, String.class, ignoreMissing); - final DomainInfo info = getRegisteredDomain(fqdn); - if (info == null) { - if (ignoreMissing) { - return document; - } else { - throw new IllegalArgumentException("unable to set domain information for document"); - } - } String fieldPrefix = targetField; if (fieldPrefix.isEmpty() == false) { fieldPrefix += "."; } - String domainTarget = fieldPrefix + "domain"; - String registeredDomainTarget = fieldPrefix + "registered_domain"; - String subdomainTarget = fieldPrefix + "subdomain"; - String topLevelDomainTarget = fieldPrefix + "top_level_domain"; - - if (info.domain() != null) { - document.setFieldValue(domainTarget, info.domain()); - } - if (info.registeredDomain() != null) { - document.setFieldValue(registeredDomainTarget, info.registeredDomain()); - } - if (info.eTLD() != null) { - document.setFieldValue(topLevelDomainTarget, info.eTLD()); - } - if (info.subdomain() != null) { - document.setFieldValue(subdomainTarget, info.subdomain()); + boolean infoFound = RegisteredDomain.parseRegisteredDomainInfo( + fqdn, + new IngestDocumentRegisteredDomainInfoCollector(document, fieldPrefix) + ); + if (infoFound == false && ignoreMissing == false) { + throw new IllegalArgumentException("unable to set domain information for document"); } return document; } - @Nullable - // visible for testing - static DomainInfo getRegisteredDomain(@Nullable String fqdn) { - if (Strings.hasText(fqdn) == false) { - return null; - } - String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); - if (registeredDomain == null) { - if (SUFFIX_MATCHER.matches(fqdn)) { - return DomainInfo.of(fqdn); - } - return null; - } - if (registeredDomain.indexOf('.') == -1) { - // we have domain with no matching public suffix, but "." in it - return null; - } - return DomainInfo.of(registeredDomain, fqdn); - } - @Override public String getType() { return TYPE; } - // visible for testing - record DomainInfo( - String domain, - String registeredDomain, - String eTLD, // n.b. https://developer.mozilla.org/en-US/docs/Glossary/eTLD - String subdomain - ) { - static DomainInfo of(final String eTLD) { - return new DomainInfo(eTLD, null, eTLD, null); - } - - static DomainInfo of(final String registeredDomain, final String domain) { - int index = registeredDomain.indexOf('.') + 1; - if (index > 0 && index < registeredDomain.length()) { - int subdomainIndex = domain.lastIndexOf("." + registeredDomain); - final String subdomain = subdomainIndex > 0 ? domain.substring(0, subdomainIndex) : null; - return new DomainInfo(domain, registeredDomain, registeredDomain.substring(index), subdomain); - } else { - return new DomainInfo(null, null, null, null); - } - } - } - public static final class Factory implements Processor.Factory { static final String DEFAULT_TARGET_FIELD = ""; @@ -151,4 +86,34 @@ public RegisteredDomainProcessor create( return new RegisteredDomainProcessor(tag, description, field, targetField, ignoreMissing); } } + + private static class IngestDocumentRegisteredDomainInfoCollector implements RegisteredDomain.RegisteredDomainInfoCollector { + private final IngestDocument document; + private final String fieldPrefix; + + IngestDocumentRegisteredDomainInfoCollector(IngestDocument document, String fieldPrefix) { + this.document = document; + this.fieldPrefix = fieldPrefix; + } + + @Override + public void domain(String domain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.DOMAIN, domain); + } + + @Override + public void registeredDomain(String registeredDomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.REGISTERED_DOMAIN, registeredDomain); + } + + @Override + public void topLevelDomain(String topLevelDomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.eTLD, topLevelDomain); + } + + @Override + public void subdomain(String subdomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.SUBDOMAIN, subdomain); + } + } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java index e1383c1dc89bb..d05a188deb559 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -10,17 +10,12 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.web.UriParts; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; import java.util.Map; public class UriPartsProcessor extends AbstractProcessor { @@ -90,87 +85,7 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { } public static Map apply(String urlString) { - URI uri = null; - URL url = null; - try { - uri = new URI(urlString); - } catch (URISyntaxException e) { - try { - url = new URL(urlString); - } catch (MalformedURLException e2) { - throw new IllegalArgumentException("unable to parse URI [" + urlString + "]"); - } - } - return getUriParts(uri, url); - } - - @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") - private static Map getUriParts(URI uri, URL fallbackUrl) { - var uriParts = new HashMap(); - String domain; - String fragment; - String path; - int port; - String query; - String scheme; - String userInfo; - - if (uri != null) { - domain = uri.getHost(); - fragment = uri.getFragment(); - path = uri.getPath(); - port = uri.getPort(); - query = uri.getQuery(); - scheme = uri.getScheme(); - userInfo = uri.getUserInfo(); - } else if (fallbackUrl != null) { - domain = fallbackUrl.getHost(); - fragment = fallbackUrl.getRef(); - path = fallbackUrl.getPath(); - port = fallbackUrl.getPort(); - query = fallbackUrl.getQuery(); - scheme = fallbackUrl.getProtocol(); - userInfo = fallbackUrl.getUserInfo(); - } else { - // should never occur during processor execution - throw new IllegalArgumentException("at least one argument must be non-null"); - } - - uriParts.put("domain", domain); - if (fragment != null) { - uriParts.put("fragment", fragment); - } - if (path != null) { - uriParts.put("path", path); - // To avoid any issues with extracting the extension from a path that contains a dot, we explicitly extract the extension - // from the last segment in the path. - var lastSegmentIndex = path.lastIndexOf('/'); - if (lastSegmentIndex >= 0) { - var lastSegment = path.substring(lastSegmentIndex); - int periodIndex = lastSegment.lastIndexOf('.'); - if (periodIndex >= 0) { - // Don't include the dot in the extension field. - uriParts.put("extension", lastSegment.substring(periodIndex + 1)); - } - } - } - if (port != -1) { - uriParts.put("port", port); - } - if (query != null) { - uriParts.put("query", query); - } - uriParts.put("scheme", scheme); - if (userInfo != null) { - uriParts.put("user_info", userInfo); - if (userInfo.contains(":")) { - int colonIndex = userInfo.indexOf(':'); - uriParts.put("username", userInfo.substring(0, colonIndex)); - uriParts.put("password", colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); - } - } - - return uriParts; + return UriParts.parse(urlString); } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java index b9fe870af2385..8aa813566f46c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -11,17 +11,14 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.TestIngestDocument; -import org.elasticsearch.ingest.common.RegisteredDomainProcessor.DomainInfo; import org.elasticsearch.test.ESTestCase; import java.util.Collections; import java.util.Map; import static java.util.Map.entry; -import static org.elasticsearch.ingest.common.RegisteredDomainProcessor.getRegisteredDomain; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; /** * Test parsing of an eTLD from a FQDN. The list of eTLDs is maintained here: @@ -32,38 +29,6 @@ */ public class RegisteredDomainProcessorTests extends ESTestCase { - public void testGetRegisteredDomain() { - assertThat(getRegisteredDomain("www.google.com"), is(new DomainInfo("www.google.com", "google.com", "com", "www"))); - assertThat(getRegisteredDomain("google.com"), is(new DomainInfo("google.com", "google.com", "com", null))); - assertThat(getRegisteredDomain(null), nullValue()); - assertThat(getRegisteredDomain(""), nullValue()); - assertThat(getRegisteredDomain(" "), nullValue()); - assertThat(getRegisteredDomain("."), nullValue()); - assertThat(getRegisteredDomain("$"), nullValue()); - assertThat(getRegisteredDomain("foo.bar.baz"), nullValue()); - assertThat( - getRegisteredDomain("www.books.amazon.co.uk"), - is(new DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) - ); - // Verify "com" is returned as the eTLD, for that FQDN or subdomain - assertThat(getRegisteredDomain("com"), is(new DomainInfo("com", null, "com", null))); - assertThat(getRegisteredDomain("example.com"), is(new DomainInfo("example.com", "example.com", "com", null))); - assertThat(getRegisteredDomain("googleapis.com"), is(new DomainInfo("googleapis.com", "googleapis.com", "com", null))); - assertThat( - getRegisteredDomain("content-autofill.googleapis.com"), - is(new DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) - ); - // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain - assertThat( - getRegisteredDomain("global.ssl.fastly.net"), - is(new DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) - ); - assertThat( - getRegisteredDomain("1.www.global.ssl.fastly.net"), - is(new DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) - ); - } - public void testBasic() throws Exception { var processor = new RegisteredDomainProcessor(null, null, "input", "output", false); { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml new file mode 100644 index 0000000000000..1bac618e7b874 --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml @@ -0,0 +1,188 @@ +--- +setup: + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline" + body: + description: "Test registered_domain processor" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_true" + body: + description: "Test registered_domain processor with ignore_missing: true" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + ignore_missing: true + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_false" + body: + description: "Test registered_domain processor with ignore_missing: false" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + ignore_missing: false + - do: + indices.create: + index: "test_registered_domain" + body: + settings: + number_of_shards: 1 + mappings: + properties: + domain_input: { type: keyword } + parsed_domain: { + type: object, + properties: { + domain: { type: keyword }, + registered_domain: { type: keyword }, + top_level_domain: { type: keyword }, + subdomain: { type: keyword } + } + } + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_true" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_false" + ignore: 404 + - do: + indices.delete: + index: "test_registered_domain" + ignore: 404 + +--- +"Registered Domain Processor: Valid FQDN": + - do: + index: + index: test_registered_domain + id: "1" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "www.google.com" } + - do: + get: + index: test_registered_domain + id: "1" + - match: { _source.domain_input: "www.google.com" } + - match: { _source.parsed_domain.domain: "www.google.com" } + - match: { _source.parsed_domain.registered_domain: "google.com" } + - match: { _source.parsed_domain.top_level_domain: "com" } + - match: { _source.parsed_domain.subdomain: "www" } + +--- +"Registered Domain Processor: Valid Domain Only (no subdomain)": + - do: + index: + index: test_registered_domain + id: "2" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "example.com" } + - do: + get: + index: test_registered_domain + id: "2" + - match: { _source.domain_input: "example.com" } + - match: { _source.parsed_domain.domain: "example.com" } + - match: { _source.parsed_domain.registered_domain: "example.com" } + - match: { _source.parsed_domain.top_level_domain: "com" } + - is_false: "_source.parsed_domain.subdomain" + +--- +"Registered Domain Processor: Valid TLD Only": + - do: + index: + index: test_registered_domain + id: "3" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "com" } + - do: + get: + index: test_registered_domain + id: "3" + - match: { _source.domain_input: "com" } + - match: { _source.parsed_domain.domain: "com" } + - is_false: "_source.parsed_domain.registered_domain" + - match: { _source.parsed_domain.top_level_domain: "com" } + - is_false: "_source.parsed_domain.subdomain" + +--- +"Registered Domain Processor: Complex FQDN": + - do: + index: + index: test_registered_domain + id: "4" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "www.books.amazon.co.uk" } + - do: + get: + index: test_registered_domain + id: "4" + - match: { _source.domain_input: "www.books.amazon.co.uk" } + - match: { _source.parsed_domain.domain: "www.books.amazon.co.uk" } + - match: { _source.parsed_domain.registered_domain: "amazon.co.uk" } + - match: { _source.parsed_domain.top_level_domain: "co.uk" } + - match: { _source.parsed_domain.subdomain: "www.books" } + +--- +"Registered Domain Processor: Unmatchable FQDN": + - do: + index: + index: test_registered_domain + id: "5" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "foo.bar.baz" } + - is_false: "_source.domain_input" + - is_false: "_source.parsed_domain" # parsed_domain should not be added +--- +"Registered Domain Processor: Invalid input": + - do: + index: + index: test_registered_domain + id: "6" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "123" } + - is_false: "_source.domain_input" + - is_false: "_source.parsed_domain" # parsed_domain should not be added + +--- +"Registered Domain Processor: Field missing with ignore_missing true": + - do: + index: + index: test_registered_domain + id: "7" + pipeline: "my_registered_domain_pipeline_ignore_missing_true" + body: { some_other_field: "value" } + - do: + get: + index: test_registered_domain + id: "7" + - match: { _source.some_other_field: "value" } + - is_false: "_source.parsed_domain" # parsed_domain should not be added + +--- +"Registered Domain Processor: Field missing with ignore_missing false causes error": + - do: + catch: bad_request + index: + index: test_registered_domain + id: "8" + pipeline: "my_registered_domain_pipeline_ignore_missing_false" + body: { some_other_field: "value" } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "field [domain_input] not present as part of path [domain_input]" }