From 4bef51f6cf97b28117b37de5f1c5b8a798de6ad9 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 6 Jan 2026 14:28:20 +0200 Subject: [PATCH 01/11] Adding UriParts functionality --- libs/web-utils/build.gradle | 20 ++ libs/web-utils/src/main/java/module-info.java | 13 + .../java/org/elasticsearch/web/UriParts.java | 134 ++++++++++ .../org/elasticsearch/web/UriPartsTests.java | 233 ++++++++++++++++++ modules/ingest-common/build.gradle | 1 + .../src/main/java/module-info.java | 1 + .../ingest/common/UriPartsProcessor.java | 88 +------ 7 files changed, 404 insertions(+), 86 deletions(-) create mode 100644 libs/web-utils/build.gradle create mode 100644 libs/web-utils/src/main/java/module-info.java create mode 100644 libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java create mode 100644 libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle new file mode 100644 index 0000000000000..85eb8ce537726 --- /dev/null +++ b/libs/web-utils/build.gradle @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +dependencies { + api project(':libs:core') + + testImplementation(project(":test:framework")) { + exclude group: 'org.elasticsearch', module: 'web-utils' + } +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/web-utils/src/main/java/module-info.java b/libs/web-utils/src/main/java/module-info.java new file mode 100644 index 0000000000000..d70512bbe51e4 --- /dev/null +++ b/libs/web-utils/src/main/java/module-info.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.web { + requires org.elasticsearch.base; + exports org.elasticsearch.web; +} diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java new file mode 100644 index 0000000000000..07a38921b514a --- /dev/null +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.core.SuppressForbidden; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class UriParts { + + private static final String DOMAIN = "domain"; + private static final String FRAGMENT = "fragment"; + private static final String PATH = "path"; + private static final String PORT = "port"; + private static final String QUERY = "query"; + private static final String SCHEME = "scheme"; + private static final String USER_INFO = "user_info"; + private static final String EXTENSION = "extension"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + + private static final Map> URI_PARTS_TYPES = Map.ofEntries( + Map.entry(DOMAIN, String.class), + Map.entry(FRAGMENT, String.class), + Map.entry(PATH, String.class), + Map.entry(PORT, Integer.class), + Map.entry(QUERY, String.class), + Map.entry(SCHEME, String.class), + Map.entry(USER_INFO, String.class), + Map.entry(EXTENSION, String.class), + Map.entry(USERNAME, String.class), + Map.entry(PASSWORD, String.class) + ); + + public static Map> getUriPartsTypes() { + return URI_PARTS_TYPES; + } + + public static Map parse(String uriString) { + URI uri = null; + URL url = null; + try { + uri = new URI(uriString); + } catch (URISyntaxException e) { + try { + url = new URL(uriString); + } catch (MalformedURLException e2) { + throw new IllegalArgumentException("unable to parse URI [" + uriString + "]"); + } + } + return getUriParts(uri, url); + } + + @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") + private static Map getUriParts(URI uri, URL fallbackUrl) { + var uriParts = new HashMap(); + String domain; + String fragment; + String path; + int port; + String query; + String scheme; + String userInfo; + + if (uri != null) { + domain = uri.getHost(); + fragment = uri.getFragment(); + path = uri.getPath(); + port = uri.getPort(); + query = uri.getQuery(); + scheme = uri.getScheme(); + userInfo = uri.getUserInfo(); + } else if (fallbackUrl != null) { + domain = fallbackUrl.getHost(); + fragment = fallbackUrl.getRef(); + path = fallbackUrl.getPath(); + port = fallbackUrl.getPort(); + query = fallbackUrl.getQuery(); + scheme = fallbackUrl.getProtocol(); + userInfo = fallbackUrl.getUserInfo(); + } else { + // should never occur during processor execution + throw new IllegalArgumentException("at least one argument must be non-null"); + } + + uriParts.put(DOMAIN, domain); + if (fragment != null) { + uriParts.put(FRAGMENT, fragment); + } + if (path != null) { + uriParts.put(PATH, path); + // To avoid any issues with extracting the extension from a path that contains a dot, we explicitly extract the extension + // from the last segment in the path. + var lastSegmentIndex = path.lastIndexOf('/'); + if (lastSegmentIndex >= 0) { + var lastSegment = path.substring(lastSegmentIndex); + int periodIndex = lastSegment.lastIndexOf('.'); + if (periodIndex >= 0) { + // Don't include the dot in the extension field. + uriParts.put(EXTENSION, lastSegment.substring(periodIndex + 1)); + } + } + } + if (port != -1) { + uriParts.put(PORT, port); + } + if (query != null) { + uriParts.put(QUERY, query); + } + uriParts.put(SCHEME, scheme); + if (userInfo != null) { + uriParts.put(USER_INFO, userInfo); + if (userInfo.contains(":")) { + int colonIndex = userInfo.indexOf(':'); + uriParts.put(USERNAME, userInfo.substring(0, colonIndex)); + uriParts.put(PASSWORD, colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); + } + } + + return uriParts; + } +} diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java new file mode 100644 index 0000000000000..8a8c4c5701e77 --- /dev/null +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/UriPartsTests.java @@ -0,0 +1,233 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; + +public class UriPartsTests extends ESTestCase { + + public void testUriParts() { + + // simple URI + testUriParsing("http://www.google.com", Map.of("scheme", "http", "domain", "www.google.com", "path", "")); + + // custom port + testUriParsing("http://www.google.com:88", Map.of("scheme", "http", "domain", "www.google.com", "path", "", "port", 88)); + + // file + testUriParsing( + "http://www.google.com:88/google.png", + Map.of("scheme", "http", "domain", "www.google.com", "extension", "png", "path", "/google.png", "port", 88) + ); + + // fragment + testUriParsing( + "https://www.google.com:88/foo#bar", + Map.of("scheme", "https", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + + // path, extension + testUriParsing( + "https://www.google.com:88/foo.jpg", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo.jpg", "extension", "jpg", "port", 88) + ); + + // query + testUriParsing( + "https://www.google.com:88/foo?key=val", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo", "query", "key=val", "port", 88) + ); + + // user_info + testUriParsing( + "https://user:pw@www.google.com:88/foo", + Map.of( + "scheme", + "https", + "domain", + "www.google.com", + "path", + "/foo", + "port", + 88, + "user_info", + "user:pw", + "username", + "user", + "password", + "pw" + ) + ); + + // user_info without password + testUriParsing( + "https://user:@www.google.com:88/foo", + Map.of( + "scheme", + "https", + "domain", + "www.google.com", + "path", + "/foo", + "port", + 88, + "user_info", + "user:", + "username", + "user", + "password", + "" + ) + ); + + // everything! + testUriParsing( + "https://user:pw@testing.google.com:8080/foo/bar?foo1=bar1&foo2=bar2#anchorVal", + Map.of( + "scheme", + "https", + "domain", + "testing.google.com", + "fragment", + "anchorVal", + "path", + "/foo/bar", + "port", + 8080, + "username", + "user", + "password", + "pw", + "user_info", + "user:pw", + "query", + "foo1=bar1&foo2=bar2" + ) + ); + + // non-http schemes + testUriParsing( + "ftp://ftp.is.co.za/rfc/rfc1808.txt", + Map.of("scheme", "ftp", "path", "/rfc/rfc1808.txt", "extension", "txt", "domain", "ftp.is.co.za") + ); + + testUriParsing("telnet://192.0.2.16:80/", Map.of("scheme", "telnet", "path", "/", "port", 80, "domain", "192.0.2.16")); + + testUriParsing( + "ldap://[2001:db8::7]/c=GB?objectClass?one", + Map.of("scheme", "ldap", "path", "/c=GB", "query", "objectClass?one", "domain", "[2001:db8::7]") + ); + + // keep original + testUriParsing( + "http://www.google.com:88/foo#bar", + Map.of("scheme", "http", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + + // remove if successful + testUriParsing( + "http://www.google.com:88/foo#bar", + Map.of("scheme", "http", "domain", "www.google.com", "fragment", "bar", "path", "/foo", "port", 88) + ); + } + + public void testUrlWithCharactersNotToleratedByUri() throws Exception { + testUriParsing( + "http://www.google.com/path with spaces", + Map.of("scheme", "http", "domain", "www.google.com", "path", "/path with spaces") + ); + + testUriParsing( + "https://user:pw@testing.google.com:8080/foo with space/bar?foo1=bar1&foo2=bar2#anchorVal", + Map.of( + "scheme", + "https", + "domain", + "testing.google.com", + "fragment", + "anchorVal", + "path", + "/foo with space/bar", + "port", + 8080, + "username", + "user", + "password", + "pw", + "user_info", + "user:pw", + "query", + "foo1=bar1&foo2=bar2" + ) + ); + } + + public void testDotPathWithoutExtension() throws Exception { + testUriParsing( + "https://www.google.com/path.withdot/filenamewithoutextension", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/path.withdot/filenamewithoutextension") + ); + } + + public void testDotPathWithExtension() throws Exception { + testUriParsing( + "https://www.google.com/path.withdot/filenamewithextension.txt", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/path.withdot/filenamewithextension.txt", "extension", "txt") + ); + } + + /** + * This test verifies that we return an empty extension instead of null if the URI ends with a period. This is probably + * not behaviour we necessarily want to keep forever, but this test ensures that we're conscious about changing that behaviour. + */ + public void testEmptyExtension() throws Exception { + testUriParsing( + "https://www.google.com/foo/bar.", + Map.of("scheme", "https", "domain", "www.google.com", "path", "/foo/bar.", "extension", "") + ); + } + + public void testInvalidUri() { + final String uri = "not:\\/_a_valid_uri"; + expectThrows(IllegalArgumentException.class, containsString("unable to parse URI [" + uri + "]"), () -> UriParts.parse(uri)); + } + + public void testNullValue() { + expectThrows(NullPointerException.class, () -> UriParts.parse(null)); + } + + private void testUriParsing(String uri, Map expectedValues) { + + Map actualValues = UriParts.parse(uri); + Map> expectedTypes = UriParts.getUriPartsTypes(); + + for (Map.Entry entry : expectedValues.entrySet()) { + String partName = entry.getKey(); + assertThat(actualValues, hasEntry(partName, entry.getValue())); + + Class expectedType = expectedTypes.get(partName); + assertNotNull("No type defined for key '" + partName + "'", expectedType); + assertTrue("Type mismatch for key '" + partName + "' for URI: " + uri, expectedType.isInstance(actualValues.get(partName))); + } + + // ensure that every key returned by parse() has a corresponding type definition in getUriPartsTypes() + for (Map.Entry entry : actualValues.entrySet()) { + assertTrue( + "Key '" + entry.getKey() + "' from parsed URI '" + uri + "' is not defined in UriParts.getUriPartsTypes()", + expectedTypes.containsKey(entry.getKey()) + ); + } + } +} diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 74d753af69325..9238ba1b2e577 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -22,6 +22,7 @@ dependencies { compileOnly project(':modules:lang-painless:spi') api project(':libs:grok') api project(':libs:dissect') + api project(':libs:web-utils') implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" } diff --git a/modules/ingest-common/src/main/java/module-info.java b/modules/ingest-common/src/main/java/module-info.java index c3b3ab90892d9..8ff0d6f03fb54 100644 --- a/modules/ingest-common/src/main/java/module-info.java +++ b/modules/ingest-common/src/main/java/module-info.java @@ -14,6 +14,7 @@ requires org.elasticsearch.painless.spi; requires org.elasticsearch.server; requires org.elasticsearch.xcontent; + requires org.elasticsearch.web; requires org.apache.httpcomponents.httpclient; requires org.apache.logging.log4j; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java index e1383c1dc89bb..814492c3c0e95 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -10,17 +10,12 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.web.UriParts; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; import java.util.Map; public class UriPartsProcessor extends AbstractProcessor { @@ -90,88 +85,9 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { } public static Map apply(String urlString) { - URI uri = null; - URL url = null; - try { - uri = new URI(urlString); - } catch (URISyntaxException e) { - try { - url = new URL(urlString); - } catch (MalformedURLException e2) { - throw new IllegalArgumentException("unable to parse URI [" + urlString + "]"); - } - } - return getUriParts(uri, url); + return UriParts.parse(urlString); } - @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") - private static Map getUriParts(URI uri, URL fallbackUrl) { - var uriParts = new HashMap(); - String domain; - String fragment; - String path; - int port; - String query; - String scheme; - String userInfo; - - if (uri != null) { - domain = uri.getHost(); - fragment = uri.getFragment(); - path = uri.getPath(); - port = uri.getPort(); - query = uri.getQuery(); - scheme = uri.getScheme(); - userInfo = uri.getUserInfo(); - } else if (fallbackUrl != null) { - domain = fallbackUrl.getHost(); - fragment = fallbackUrl.getRef(); - path = fallbackUrl.getPath(); - port = fallbackUrl.getPort(); - query = fallbackUrl.getQuery(); - scheme = fallbackUrl.getProtocol(); - userInfo = fallbackUrl.getUserInfo(); - } else { - // should never occur during processor execution - throw new IllegalArgumentException("at least one argument must be non-null"); - } - - uriParts.put("domain", domain); - if (fragment != null) { - uriParts.put("fragment", fragment); - } - if (path != null) { - uriParts.put("path", path); - // To avoid any issues with extracting the extension from a path that contains a dot, we explicitly extract the extension - // from the last segment in the path. - var lastSegmentIndex = path.lastIndexOf('/'); - if (lastSegmentIndex >= 0) { - var lastSegment = path.substring(lastSegmentIndex); - int periodIndex = lastSegment.lastIndexOf('.'); - if (periodIndex >= 0) { - // Don't include the dot in the extension field. - uriParts.put("extension", lastSegment.substring(periodIndex + 1)); - } - } - } - if (port != -1) { - uriParts.put("port", port); - } - if (query != null) { - uriParts.put("query", query); - } - uriParts.put("scheme", scheme); - if (userInfo != null) { - uriParts.put("user_info", userInfo); - if (userInfo.contains(":")) { - int colonIndex = userInfo.indexOf(':'); - uriParts.put("username", userInfo.substring(0, colonIndex)); - uriParts.put("password", colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); - } - } - - return uriParts; - } @Override public String getType() { From 7910794251f04363fccaef4226ca5ca364b4dbd2 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 6 Jan 2026 14:29:28 +0200 Subject: [PATCH 02/11] spotless --- libs/web-utils/src/main/java/module-info.java | 1 + .../java/org/elasticsearch/ingest/common/UriPartsProcessor.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/web-utils/src/main/java/module-info.java b/libs/web-utils/src/main/java/module-info.java index d70512bbe51e4..be0010069606d 100644 --- a/libs/web-utils/src/main/java/module-info.java +++ b/libs/web-utils/src/main/java/module-info.java @@ -9,5 +9,6 @@ module org.elasticsearch.web { requires org.elasticsearch.base; + exports org.elasticsearch.web; } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java index 814492c3c0e95..d05a188deb559 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -88,7 +88,6 @@ public static Map apply(String urlString) { return UriParts.parse(urlString); } - @Override public String getType() { return TYPE; From 65beddae101ce10544b6e69c5238a980ff8b52a1 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 7 Jan 2026 16:20:37 +0200 Subject: [PATCH 03/11] Refactoring registered_domain --- libs/web-utils/build.gradle | 2 + libs/web-utils/src/main/java/module-info.java | 2 + .../elasticsearch/web/RegisteredDomain.java | 60 ++++++ .../web/RegisteredDomainTests.java | 57 ++++++ modules/ingest-common/build.gradle | 2 - .../src/main/java/module-info.java | 1 - .../common/RegisteredDomainProcessor.java | 51 +---- .../RegisteredDomainProcessorTests.java | 35 ---- .../390_registered_domain_processor.yml | 189 ++++++++++++++++++ .../bootstrap/TestScopeResolver.java | 8 + 10 files changed, 320 insertions(+), 87 deletions(-) create mode 100644 libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java create mode 100644 libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java create mode 100644 modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle index 85eb8ce537726..74240cc9086d4 100644 --- a/libs/web-utils/build.gradle +++ b/libs/web-utils/build.gradle @@ -9,6 +9,8 @@ dependencies { api project(':libs:core') + implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" + implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'web-utils' diff --git a/libs/web-utils/src/main/java/module-info.java b/libs/web-utils/src/main/java/module-info.java index be0010069606d..319f627d093c2 100644 --- a/libs/web-utils/src/main/java/module-info.java +++ b/libs/web-utils/src/main/java/module-info.java @@ -9,6 +9,8 @@ module org.elasticsearch.web { requires org.elasticsearch.base; + requires org.apache.httpcomponents.httpclient; + requires org.apache.httpcomponents.httpcore; exports org.elasticsearch.web; } diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java new file mode 100644 index 0000000000000..2979c9523b191 --- /dev/null +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.elasticsearch.core.Nullable; + +public class RegisteredDomain { + + private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); + + @Nullable + public static DomainInfo getRegisteredDomain(@Nullable String fqdn) { + if (fqdn == null || fqdn.isBlank()) { + return null; + } + String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); + if (registeredDomain == null) { + if (SUFFIX_MATCHER.matches(fqdn)) { + return DomainInfo.of(fqdn); + } + return null; + } + if (registeredDomain.indexOf('.') == -1) { + // we have domain with no matching public suffix, but "." in it + return null; + } + return DomainInfo.of(registeredDomain, fqdn); + } + + public record DomainInfo( + String domain, + String registeredDomain, + String eTLD, // n.b. https://developer.mozilla.org/en-US/docs/Glossary/eTLD + String subdomain + ) { + public static DomainInfo of(final String eTLD) { + return new DomainInfo(eTLD, null, eTLD, null); + } + + public static DomainInfo of(final String registeredDomain, final String domain) { + int index = registeredDomain.indexOf('.') + 1; + if (index > 0 && index < registeredDomain.length()) { + int subdomainIndex = domain.lastIndexOf("." + registeredDomain); + final String subdomain = subdomainIndex > 0 ? domain.substring(0, subdomainIndex) : null; + return new DomainInfo(domain, registeredDomain, registeredDomain.substring(index), subdomain); + } else { + return new DomainInfo(null, null, null, null); + } + } + } +} diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java new file mode 100644 index 0000000000000..d850ff386cb1c --- /dev/null +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.web; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.web.RegisteredDomain.getRegisteredDomain; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class RegisteredDomainTests extends ESTestCase { + + public void testGetRegisteredDomain() { + assertThat( + getRegisteredDomain("www.google.com"), + is(new RegisteredDomain.DomainInfo("www.google.com", "google.com", "com", "www")) + ); + assertThat(getRegisteredDomain("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); + assertThat(getRegisteredDomain(null), nullValue()); + assertThat(getRegisteredDomain(""), nullValue()); + assertThat(getRegisteredDomain(" "), nullValue()); + assertThat(getRegisteredDomain("."), nullValue()); + assertThat(getRegisteredDomain("$"), nullValue()); + assertThat(getRegisteredDomain("foo.bar.baz"), nullValue()); + assertThat( + getRegisteredDomain("www.books.amazon.co.uk"), + is(new RegisteredDomain.DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) + ); + // Verify "com" is returned as the eTLD, for that FQDN or subdomain + assertThat(getRegisteredDomain("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); + assertThat(getRegisteredDomain("example.com"), is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null))); + assertThat( + getRegisteredDomain("googleapis.com"), + is(new RegisteredDomain.DomainInfo("googleapis.com", "googleapis.com", "com", null)) + ); + assertThat( + getRegisteredDomain("content-autofill.googleapis.com"), + is(new RegisteredDomain.DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) + ); + // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain + assertThat( + getRegisteredDomain("global.ssl.fastly.net"), + is(new RegisteredDomain.DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) + ); + assertThat( + getRegisteredDomain("1.www.global.ssl.fastly.net"), + is(new RegisteredDomain.DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) + ); + } +} diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 9238ba1b2e577..5df689a88b93e 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -23,8 +23,6 @@ dependencies { api project(':libs:grok') api project(':libs:dissect') api project(':libs:web-utils') - implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" - implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" } restResources { diff --git a/modules/ingest-common/src/main/java/module-info.java b/modules/ingest-common/src/main/java/module-info.java index 8ff0d6f03fb54..2972996613b0d 100644 --- a/modules/ingest-common/src/main/java/module-info.java +++ b/modules/ingest-common/src/main/java/module-info.java @@ -16,7 +16,6 @@ requires org.elasticsearch.xcontent; requires org.elasticsearch.web; - requires org.apache.httpcomponents.httpclient; requires org.apache.logging.log4j; requires org.apache.lucene.analysis.common; requires org.jruby.joni; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index fa6df0c7b4f78..bef144ad78906 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -9,22 +9,18 @@ package org.elasticsearch.ingest.common; -import org.apache.http.conn.util.PublicSuffixMatcher; -import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.elasticsearch.cluster.metadata.ProjectId; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Nullable; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.web.RegisteredDomain; import java.util.Map; public class RegisteredDomainProcessor extends AbstractProcessor { public static final String TYPE = "registered_domain"; - private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); private final String field; private final String targetField; @@ -52,7 +48,7 @@ public boolean getIgnoreMissing() { @Override public IngestDocument execute(IngestDocument document) throws Exception { final String fqdn = document.getFieldValue(field, String.class, ignoreMissing); - final DomainInfo info = getRegisteredDomain(fqdn); + final RegisteredDomain.DomainInfo info = RegisteredDomain.getRegisteredDomain(fqdn); if (info == null) { if (ignoreMissing) { return document; @@ -84,54 +80,11 @@ public IngestDocument execute(IngestDocument document) throws Exception { return document; } - @Nullable - // visible for testing - static DomainInfo getRegisteredDomain(@Nullable String fqdn) { - if (Strings.hasText(fqdn) == false) { - return null; - } - String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); - if (registeredDomain == null) { - if (SUFFIX_MATCHER.matches(fqdn)) { - return DomainInfo.of(fqdn); - } - return null; - } - if (registeredDomain.indexOf('.') == -1) { - // we have domain with no matching public suffix, but "." in it - return null; - } - return DomainInfo.of(registeredDomain, fqdn); - } - @Override public String getType() { return TYPE; } - // visible for testing - record DomainInfo( - String domain, - String registeredDomain, - String eTLD, // n.b. https://developer.mozilla.org/en-US/docs/Glossary/eTLD - String subdomain - ) { - static DomainInfo of(final String eTLD) { - return new DomainInfo(eTLD, null, eTLD, null); - } - - static DomainInfo of(final String registeredDomain, final String domain) { - int index = registeredDomain.indexOf('.') + 1; - if (index > 0 && index < registeredDomain.length()) { - int subdomainIndex = domain.lastIndexOf("." + registeredDomain); - final String subdomain = subdomainIndex > 0 ? domain.substring(0, subdomainIndex) : null; - return new DomainInfo(domain, registeredDomain, registeredDomain.substring(index), subdomain); - } else { - return new DomainInfo(null, null, null, null); - } - } - } - public static final class Factory implements Processor.Factory { static final String DEFAULT_TARGET_FIELD = ""; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java index b9fe870af2385..8aa813566f46c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -11,17 +11,14 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.TestIngestDocument; -import org.elasticsearch.ingest.common.RegisteredDomainProcessor.DomainInfo; import org.elasticsearch.test.ESTestCase; import java.util.Collections; import java.util.Map; import static java.util.Map.entry; -import static org.elasticsearch.ingest.common.RegisteredDomainProcessor.getRegisteredDomain; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; /** * Test parsing of an eTLD from a FQDN. The list of eTLDs is maintained here: @@ -32,38 +29,6 @@ */ public class RegisteredDomainProcessorTests extends ESTestCase { - public void testGetRegisteredDomain() { - assertThat(getRegisteredDomain("www.google.com"), is(new DomainInfo("www.google.com", "google.com", "com", "www"))); - assertThat(getRegisteredDomain("google.com"), is(new DomainInfo("google.com", "google.com", "com", null))); - assertThat(getRegisteredDomain(null), nullValue()); - assertThat(getRegisteredDomain(""), nullValue()); - assertThat(getRegisteredDomain(" "), nullValue()); - assertThat(getRegisteredDomain("."), nullValue()); - assertThat(getRegisteredDomain("$"), nullValue()); - assertThat(getRegisteredDomain("foo.bar.baz"), nullValue()); - assertThat( - getRegisteredDomain("www.books.amazon.co.uk"), - is(new DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) - ); - // Verify "com" is returned as the eTLD, for that FQDN or subdomain - assertThat(getRegisteredDomain("com"), is(new DomainInfo("com", null, "com", null))); - assertThat(getRegisteredDomain("example.com"), is(new DomainInfo("example.com", "example.com", "com", null))); - assertThat(getRegisteredDomain("googleapis.com"), is(new DomainInfo("googleapis.com", "googleapis.com", "com", null))); - assertThat( - getRegisteredDomain("content-autofill.googleapis.com"), - is(new DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) - ); - // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain - assertThat( - getRegisteredDomain("global.ssl.fastly.net"), - is(new DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) - ); - assertThat( - getRegisteredDomain("1.www.global.ssl.fastly.net"), - is(new DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) - ); - } - public void testBasic() throws Exception { var processor = new RegisteredDomainProcessor(null, null, "input", "output", false); { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml new file mode 100644 index 0000000000000..6b38f8907a4ed --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml @@ -0,0 +1,189 @@ +--- +setup: + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline" + body: + description: "Test registered_domain processor" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_true" + body: + description: "Test registered_domain processor with ignore_missing: true" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + ignore_missing: true + - do: + ingest.put_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_false" + body: + description: "Test registered_domain processor with ignore_missing: false" + processors: + - registered_domain: + field: "domain_input" + target_field: "parsed_domain" + ignore_missing: false + - do: + indices.create: + index: "test_registered_domain" + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + domain_input: { type: keyword } + parsed_domain: { + type: object, + properties: { + domain: { type: keyword }, + registered_domain: { type: keyword }, + top_level_domain: { type: keyword }, + subdomain: { type: keyword } + } + } + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_true" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "my_registered_domain_pipeline_ignore_missing_false" + ignore: 404 + - do: + indices.delete: + index: "test_registered_domain" + ignore: 404 + +--- +"Registered Domain Processor: Valid FQDN": + - do: + index: + index: test_registered_domain + id: "1" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "www.google.com" } + - do: + get: + index: test_registered_domain + id: "1" + - match: { _source.domain_input: "www.google.com" } + - match: { _source.parsed_domain.domain: "www.google.com" } + - match: { _source.parsed_domain.registered_domain: "google.com" } + - match: { _source.parsed_domain.top_level_domain: "com" } + - match: { _source.parsed_domain.subdomain: "www" } + +--- +"Registered Domain Processor: Valid Domain Only (no subdomain)": + - do: + index: + index: test_registered_domain + id: "2" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "example.com" } + - do: + get: + index: test_registered_domain + id: "2" + - match: { _source.domain_input: "example.com" } + - match: { _source.parsed_domain.domain: "example.com" } + - match: { _source.parsed_domain.registered_domain: "example.com" } + - match: { _source.parsed_domain.top_level_domain: "com" } + - is_false: "_source.parsed_domain.subdomain" + +--- +"Registered Domain Processor: Valid TLD Only": + - do: + index: + index: test_registered_domain + id: "3" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "com" } + - do: + get: + index: test_registered_domain + id: "3" + - match: { _source.domain_input: "com" } + - match: { _source.parsed_domain.domain: "com" } + - is_false: "_source.parsed_domain.registered_domain" + - match: { _source.parsed_domain.top_level_domain: "com" } + - is_false: "_source.parsed_domain.subdomain" + +--- +"Registered Domain Processor: Complex FQDN": + - do: + index: + index: test_registered_domain + id: "4" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "www.books.amazon.co.uk" } + - do: + get: + index: test_registered_domain + id: "4" + - match: { _source.domain_input: "www.books.amazon.co.uk" } + - match: { _source.parsed_domain.domain: "www.books.amazon.co.uk" } + - match: { _source.parsed_domain.registered_domain: "amazon.co.uk" } + - match: { _source.parsed_domain.top_level_domain: "co.uk" } + - match: { _source.parsed_domain.subdomain: "www.books" } + +--- +"Registered Domain Processor: Unmatchable FQDN": + - do: + index: + index: test_registered_domain + id: "5" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "foo.bar.baz" } + - is_false: "_source.domain_input" + - is_false: "_source.parsed_domain" # parsed_domain should not be added +--- +"Registered Domain Processor: Invalid input": + - do: + index: + index: test_registered_domain + id: "6" + pipeline: "my_registered_domain_pipeline" + body: { domain_input: "123" } + - is_false: "_source.domain_input" + - is_false: "_source.parsed_domain" # parsed_domain should not be added + +--- +"Registered Domain Processor: Field missing with ignore_missing true": + - do: + index: + index: test_registered_domain + id: "7" + pipeline: "my_registered_domain_pipeline_ignore_missing_true" + body: { some_other_field: "value" } + - do: + get: + index: test_registered_domain + id: "7" + - match: { _source.some_other_field: "value" } + - is_false: "_source.parsed_domain" # parsed_domain should not be added + +--- +"Registered Domain Processor: Field missing with ignore_missing false causes error": + - do: + catch: bad_request + index: + index: test_registered_domain + id: "8" + pipeline: "my_registered_domain_pipeline_ignore_missing_false" + body: { some_other_field: "value" } + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "field [domain_input] not present as part of path [domain_input]" } diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java index 91662f3f35773..9ee54f712fbb1 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java @@ -82,6 +82,14 @@ PolicyScope getScope(Class callerClass) { if (callerClass.getPackageName().startsWith("org.bouncycastle")) { scope = new PolicyScope(PLUGIN, "security", ALL_UNNAMED); logger.debug("Assuming bouncycastle is part of the security plugin"); + } else if (callerClass.getPackageName().startsWith("org.apache.httpcomponents") + || callerClass.getPackageName().startsWith("org.apache.http.conn")) { + String moduleName = callerClass.getModule().getName(); + if (moduleName == null) { + moduleName = ALL_UNNAMED; + } + scope = new PolicyScope(PLUGIN, "ingest-common/esql", moduleName); + logger.debug("Assuming httpclient is related to the ingest-common and esql plugins for tests"); } } if (scope == null) { From 79d4cee5ac8399aa75868cade965f49367dfccd6 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 7 Jan 2026 16:23:55 +0200 Subject: [PATCH 04/11] spotless --- .../java/org/elasticsearch/bootstrap/TestScopeResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java index 9ee54f712fbb1..765261fc664e1 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java @@ -90,7 +90,7 @@ PolicyScope getScope(Class callerClass) { } scope = new PolicyScope(PLUGIN, "ingest-common/esql", moduleName); logger.debug("Assuming httpclient is related to the ingest-common and esql plugins for tests"); - } + } } if (scope == null) { logger.warn("Cannot identify a scope for class [{}], location [{}]", callerClass.getName(), location); From 0b6df99c52b094638e8ff026f04085fbb1f61be3 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 7 Jan 2026 16:39:05 +0200 Subject: [PATCH 05/11] Moving licences directory from ingest-common to web-utils --- .../web-utils}/licenses/httpclient-LICENSE.txt | 0 .../web-utils}/licenses/httpclient-NOTICE.txt | 0 .../web-utils}/licenses/httpcore-LICENSE.txt | 0 .../ingest-common => libs/web-utils}/licenses/httpcore-NOTICE.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {modules/ingest-common => libs/web-utils}/licenses/httpclient-LICENSE.txt (100%) rename {modules/ingest-common => libs/web-utils}/licenses/httpclient-NOTICE.txt (100%) rename {modules/ingest-common => libs/web-utils}/licenses/httpcore-LICENSE.txt (100%) rename {modules/ingest-common => libs/web-utils}/licenses/httpcore-NOTICE.txt (100%) diff --git a/modules/ingest-common/licenses/httpclient-LICENSE.txt b/libs/web-utils/licenses/httpclient-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/httpclient-LICENSE.txt rename to libs/web-utils/licenses/httpclient-LICENSE.txt diff --git a/modules/ingest-common/licenses/httpclient-NOTICE.txt b/libs/web-utils/licenses/httpclient-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/httpclient-NOTICE.txt rename to libs/web-utils/licenses/httpclient-NOTICE.txt diff --git a/modules/ingest-common/licenses/httpcore-LICENSE.txt b/libs/web-utils/licenses/httpcore-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/httpcore-LICENSE.txt rename to libs/web-utils/licenses/httpcore-LICENSE.txt diff --git a/modules/ingest-common/licenses/httpcore-NOTICE.txt b/libs/web-utils/licenses/httpcore-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/httpcore-NOTICE.txt rename to libs/web-utils/licenses/httpcore-NOTICE.txt From 2031df535534b8fecf604c58789dfdaf97242cd2 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 7 Jan 2026 17:46:58 +0200 Subject: [PATCH 06/11] Ignore missing classes in thirdPartyAudit --- libs/web-utils/build.gradle | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle index 74240cc9086d4..1efe7c32e09a5 100644 --- a/libs/web-utils/build.gradle +++ b/libs/web-utils/build.gradle @@ -20,3 +20,12 @@ dependencies { tasks.named('forbiddenApisMain').configure { replaceSignatureFiles 'jdk-signatures' } + +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses( + //commons-logging + 'org.apache.commons.codec.binary.Base64', + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + ) +} From 71ccbd3a8c35e226b59e7425ea4223112defc26c Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 8 Jan 2026 17:14:11 +0200 Subject: [PATCH 07/11] Refactor RegisteredDomain result to be a Map --- .../elasticsearch/web/RegisteredDomain.java | 123 ++++++++++++++++-- .../web/RegisteredDomainTests.java | 32 ++--- .../common/RegisteredDomainProcessor.java | 26 ++-- 3 files changed, 143 insertions(+), 38 deletions(-) diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java index 2979c9523b191..5e980a8361c7d 100644 --- a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java @@ -13,12 +13,34 @@ import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.elasticsearch.core.Nullable; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + public class RegisteredDomain { + public static final String DOMAIN = "domain"; + public static final String REGISTERED_DOMAIN = "registered_domain"; + public static final String ETLD = "top_level_domain"; + public static final String SUBDOMAIN = "subdomain"; + + public static final LinkedHashMap> OUTPUT_FIELDS; + private static final Set OUTPUT_FIELD_KEYS; + + static { + OUTPUT_FIELDS = new LinkedHashMap<>(); + OUTPUT_FIELDS.putLast(DOMAIN, String.class); + OUTPUT_FIELDS.putLast(REGISTERED_DOMAIN, String.class); + OUTPUT_FIELDS.putLast(ETLD, String.class); + OUTPUT_FIELDS.putLast(SUBDOMAIN, String.class); + OUTPUT_FIELD_KEYS = OUTPUT_FIELDS.keySet(); + } + private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); @Nullable - public static DomainInfo getRegisteredDomain(@Nullable String fqdn) { + public static Map getRegisteredDomainInfo(@Nullable String fqdn) { if (fqdn == null || fqdn.isBlank()) { return null; } @@ -36,17 +58,26 @@ public static DomainInfo getRegisteredDomain(@Nullable String fqdn) { return DomainInfo.of(registeredDomain, fqdn); } - public record DomainInfo( - String domain, - String registeredDomain, - String eTLD, // n.b. https://developer.mozilla.org/en-US/docs/Glossary/eTLD - String subdomain - ) { - public static DomainInfo of(final String eTLD) { + public static Map> getRegisteredDomainInfoFields() { + return OUTPUT_FIELDS; + } + + /** + * A map implementation for the domain info that does not incur lookup overhead like a {@code HashMap} for example. + * Keys are known in advance and lookup relies on an efficient switch statement rather than {@code hashCode()} and {@code equals()}. + * + * @param domain the domain name + * @param registeredDomain the registered domain name + * @param eTLD n.b. eTLD + * @param subdomain the subdomain name + */ + @SuppressWarnings("NullableProblems") + record DomainInfo(String domain, String registeredDomain, String eTLD, String subdomain) implements Map { + static DomainInfo of(final String eTLD) { return new DomainInfo(eTLD, null, eTLD, null); } - public static DomainInfo of(final String registeredDomain, final String domain) { + static DomainInfo of(final String registeredDomain, final String domain) { int index = registeredDomain.indexOf('.') + 1; if (index > 0 && index < registeredDomain.length()) { int subdomainIndex = domain.lastIndexOf("." + registeredDomain); @@ -56,5 +87,79 @@ public static DomainInfo of(final String registeredDomain, final String domain) return new DomainInfo(null, null, null, null); } } + + @Override + public int size() { + return OUTPUT_FIELDS.size(); + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public boolean containsKey(Object key) { + return OUTPUT_FIELD_KEYS.contains(key.toString()); + } + + @Override + public boolean containsValue(Object value) { + throw new UnsupportedOperationException("This map does not support containsValue"); + } + + /** + * A quick lookup that takes advantage of the fact that keys are known in advance. + * Assuming that lookup relies on the public constants above (e.g {@link #DOMAIN}), the switch statement is expected to be + * very efficient as it doesn't require computing hash codes or checking equality of arbitrary strings. + * + * @param key the key whose associated value is to be returned + * @return the value to which the specified key is mapped, or {@code null} if this map contains no mapping for the key + */ + @Override + public String get(Object key) { + return switch ((String) key) { + case DOMAIN -> domain; + case REGISTERED_DOMAIN -> registeredDomain; + case ETLD -> eTLD; + case SUBDOMAIN -> subdomain; + default -> null; + }; + } + + @Override + public String put(String key, String value) { + throw new UnsupportedOperationException("This map is populated with values at construction time"); + } + + @Override + public String remove(Object key) { + throw new UnsupportedOperationException("This map is unmodifiable"); + } + + @Override + public void putAll(Map m) { + throw new UnsupportedOperationException("This map is unmodifiable"); + } + + @Override + public void clear() { + throw new UnsupportedOperationException("This map is unmodifiable"); + } + + @Override + public Set keySet() { + return OUTPUT_FIELD_KEYS; + } + + @Override + public Collection values() { + throw new UnsupportedOperationException("This map does not support iteration over values"); + } + + @Override + public Set> entrySet() { + throw new UnsupportedOperationException("This map does not support iteration over entries"); + } } } diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java index d850ff386cb1c..e2e643882ced0 100644 --- a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.web.RegisteredDomain.getRegisteredDomain; +import static org.elasticsearch.web.RegisteredDomain.getRegisteredDomainInfo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -19,38 +19,38 @@ public class RegisteredDomainTests extends ESTestCase { public void testGetRegisteredDomain() { assertThat( - getRegisteredDomain("www.google.com"), + getRegisteredDomainInfo("www.google.com"), is(new RegisteredDomain.DomainInfo("www.google.com", "google.com", "com", "www")) ); - assertThat(getRegisteredDomain("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); - assertThat(getRegisteredDomain(null), nullValue()); - assertThat(getRegisteredDomain(""), nullValue()); - assertThat(getRegisteredDomain(" "), nullValue()); - assertThat(getRegisteredDomain("."), nullValue()); - assertThat(getRegisteredDomain("$"), nullValue()); - assertThat(getRegisteredDomain("foo.bar.baz"), nullValue()); + assertThat(getRegisteredDomainInfo("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); + assertThat(getRegisteredDomainInfo(null), nullValue()); + assertThat(getRegisteredDomainInfo(""), nullValue()); + assertThat(getRegisteredDomainInfo(" "), nullValue()); + assertThat(getRegisteredDomainInfo("."), nullValue()); + assertThat(getRegisteredDomainInfo("$"), nullValue()); + assertThat(getRegisteredDomainInfo("foo.bar.baz"), nullValue()); assertThat( - getRegisteredDomain("www.books.amazon.co.uk"), + getRegisteredDomainInfo("www.books.amazon.co.uk"), is(new RegisteredDomain.DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) ); // Verify "com" is returned as the eTLD, for that FQDN or subdomain - assertThat(getRegisteredDomain("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); - assertThat(getRegisteredDomain("example.com"), is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null))); + assertThat(getRegisteredDomainInfo("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); + assertThat(getRegisteredDomainInfo("example.com"), is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null))); assertThat( - getRegisteredDomain("googleapis.com"), + getRegisteredDomainInfo("googleapis.com"), is(new RegisteredDomain.DomainInfo("googleapis.com", "googleapis.com", "com", null)) ); assertThat( - getRegisteredDomain("content-autofill.googleapis.com"), + getRegisteredDomainInfo("content-autofill.googleapis.com"), is(new RegisteredDomain.DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) ); // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain assertThat( - getRegisteredDomain("global.ssl.fastly.net"), + getRegisteredDomainInfo("global.ssl.fastly.net"), is(new RegisteredDomain.DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) ); assertThat( - getRegisteredDomain("1.www.global.ssl.fastly.net"), + getRegisteredDomainInfo("1.www.global.ssl.fastly.net"), is(new RegisteredDomain.DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) ); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index bef144ad78906..13b7e140e37f3 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -48,7 +48,7 @@ public boolean getIgnoreMissing() { @Override public IngestDocument execute(IngestDocument document) throws Exception { final String fqdn = document.getFieldValue(field, String.class, ignoreMissing); - final RegisteredDomain.DomainInfo info = RegisteredDomain.getRegisteredDomain(fqdn); + final Map info = RegisteredDomain.getRegisteredDomainInfo(fqdn); if (info == null) { if (ignoreMissing) { return document; @@ -60,22 +60,22 @@ public IngestDocument execute(IngestDocument document) throws Exception { if (fieldPrefix.isEmpty() == false) { fieldPrefix += "."; } - String domainTarget = fieldPrefix + "domain"; - String registeredDomainTarget = fieldPrefix + "registered_domain"; - String subdomainTarget = fieldPrefix + "subdomain"; - String topLevelDomainTarget = fieldPrefix + "top_level_domain"; - if (info.domain() != null) { - document.setFieldValue(domainTarget, info.domain()); + String domain = info.get(RegisteredDomain.DOMAIN); + if (domain != null) { + document.setFieldValue(fieldPrefix + RegisteredDomain.DOMAIN, domain); } - if (info.registeredDomain() != null) { - document.setFieldValue(registeredDomainTarget, info.registeredDomain()); + String registeredDomain = info.get(RegisteredDomain.REGISTERED_DOMAIN); + if (registeredDomain != null) { + document.setFieldValue(fieldPrefix + RegisteredDomain.REGISTERED_DOMAIN, registeredDomain); } - if (info.eTLD() != null) { - document.setFieldValue(topLevelDomainTarget, info.eTLD()); + String eTLD = info.get(RegisteredDomain.ETLD); + if (eTLD != null) { + document.setFieldValue(fieldPrefix + RegisteredDomain.ETLD, eTLD); } - if (info.subdomain() != null) { - document.setFieldValue(subdomainTarget, info.subdomain()); + String subdomain = info.get(RegisteredDomain.SUBDOMAIN); + if (subdomain != null) { + document.setFieldValue(fieldPrefix + RegisteredDomain.SUBDOMAIN, subdomain); } return document; } From c5b1ea63bd33d2a03eb30f2ea79a6c3cb999bb76 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:37:11 +0200 Subject: [PATCH 08/11] Match esql output collection requirements --- .../elasticsearch/web/RegisteredDomain.java | 179 +++++++----------- .../java/org/elasticsearch/web/UriParts.java | 170 +++++++++++++---- .../web/RegisteredDomainTests.java | 35 ++-- .../common/RegisteredDomainProcessor.java | 60 +++--- 4 files changed, 266 insertions(+), 178 deletions(-) diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java index 5e980a8361c7d..9bd3b98df0480 100644 --- a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java @@ -13,10 +13,7 @@ import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.elasticsearch.core.Nullable; -import java.util.Collection; import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; public class RegisteredDomain { @@ -25,141 +22,111 @@ public class RegisteredDomain { public static final String ETLD = "top_level_domain"; public static final String SUBDOMAIN = "subdomain"; - public static final LinkedHashMap> OUTPUT_FIELDS; - private static final Set OUTPUT_FIELD_KEYS; + public static final LinkedHashMap> REGISTERED_DOMAIN_INFO_FIELDS; static { - OUTPUT_FIELDS = new LinkedHashMap<>(); - OUTPUT_FIELDS.putLast(DOMAIN, String.class); - OUTPUT_FIELDS.putLast(REGISTERED_DOMAIN, String.class); - OUTPUT_FIELDS.putLast(ETLD, String.class); - OUTPUT_FIELDS.putLast(SUBDOMAIN, String.class); - OUTPUT_FIELD_KEYS = OUTPUT_FIELDS.keySet(); + REGISTERED_DOMAIN_INFO_FIELDS = new LinkedHashMap<>(); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(DOMAIN, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(REGISTERED_DOMAIN, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(ETLD, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(SUBDOMAIN, String.class); } private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); - @Nullable - public static Map getRegisteredDomainInfo(@Nullable String fqdn) { + public static boolean parseRegisteredDomainInfo(@Nullable final String fqdn, final RegisteredDomainInfoCollector collector) { if (fqdn == null || fqdn.isBlank()) { - return null; + return false; } String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fqdn); if (registeredDomain == null) { if (SUFFIX_MATCHER.matches(fqdn)) { - return DomainInfo.of(fqdn); + collector.topLevelDomain(fqdn); + collector.domain(fqdn); + return true; } - return null; + return false; } - if (registeredDomain.indexOf('.') == -1) { - // we have domain with no matching public suffix, but "." in it - return null; + int indexOfDot = registeredDomain.indexOf('.') + 1; + if (indexOfDot > 0 && indexOfDot < registeredDomain.length()) { + collector.domain(fqdn); + collector.registeredDomain(registeredDomain); + collector.topLevelDomain(registeredDomain.substring(indexOfDot)); + int subdomainIndex = fqdn.lastIndexOf("." + registeredDomain); + if (subdomainIndex > 0) { + collector.subdomain(fqdn.substring(0, subdomainIndex)); + } + return true; } - return DomainInfo.of(registeredDomain, fqdn); + return false; } - public static Map> getRegisteredDomainInfoFields() { - return OUTPUT_FIELDS; + public static LinkedHashMap> getRegisteredDomainInfoFields() { + return REGISTERED_DOMAIN_INFO_FIELDS; } /** - * A map implementation for the domain info that does not incur lookup overhead like a {@code HashMap} for example. - * Keys are known in advance and lookup relies on an efficient switch statement rather than {@code hashCode()} and {@code equals()}. - * - * @param domain the domain name - * @param registeredDomain the registered domain name - * @param eTLD n.b. eTLD - * @param subdomain the subdomain name + * A collector for registered domain information. + * Implementation can be specific to the use case, for example - it can write parsed info directly to the collecting data structure. */ - @SuppressWarnings("NullableProblems") - record DomainInfo(String domain, String registeredDomain, String eTLD, String subdomain) implements Map { - static DomainInfo of(final String eTLD) { - return new DomainInfo(eTLD, null, eTLD, null); - } - - static DomainInfo of(final String registeredDomain, final String domain) { - int index = registeredDomain.indexOf('.') + 1; - if (index > 0 && index < registeredDomain.length()) { - int subdomainIndex = domain.lastIndexOf("." + registeredDomain); - final String subdomain = subdomainIndex > 0 ? domain.substring(0, subdomainIndex) : null; - return new DomainInfo(domain, registeredDomain, registeredDomain.substring(index), subdomain); - } else { - return new DomainInfo(null, null, null, null); - } - } - - @Override - public int size() { - return OUTPUT_FIELDS.size(); - } - - @Override - public boolean isEmpty() { - return false; - } + public interface RegisteredDomainInfoCollector { + /** + * @param domain the domain name + */ + void domain(String domain); - @Override - public boolean containsKey(Object key) { - return OUTPUT_FIELD_KEYS.contains(key.toString()); - } + /** + * @param registeredDomain the registered domain name + */ + void registeredDomain(String registeredDomain); - @Override - public boolean containsValue(Object value) { - throw new UnsupportedOperationException("This map does not support containsValue"); - } + /** + * @param topLevelDomain the top level domain, n.b. eTLD + */ + void topLevelDomain(String topLevelDomain); /** - * A quick lookup that takes advantage of the fact that keys are known in advance. - * Assuming that lookup relies on the public constants above (e.g {@link #DOMAIN}), the switch statement is expected to be - * very efficient as it doesn't require computing hash codes or checking equality of arbitrary strings. - * - * @param key the key whose associated value is to be returned - * @return the value to which the specified key is mapped, or {@code null} if this map contains no mapping for the key + * @param subdomain the subdomain name */ - @Override - public String get(Object key) { - return switch ((String) key) { - case DOMAIN -> domain; - case REGISTERED_DOMAIN -> registeredDomain; - case ETLD -> eTLD; - case SUBDOMAIN -> subdomain; - default -> null; - }; - } + void subdomain(String subdomain); + } - @Override - public String put(String key, String value) { - throw new UnsupportedOperationException("This map is populated with values at construction time"); - } + public record DomainInfo(String domain, String registeredDomain, String eTLD, String subdomain) { + static class Factory implements RegisteredDomainInfoCollector { + String domain; + String registeredDomain; + String topLevelDomain; + String subdomain; - @Override - public String remove(Object key) { - throw new UnsupportedOperationException("This map is unmodifiable"); - } + @Override + public void domain(String domain) { + this.domain = domain; + } - @Override - public void putAll(Map m) { - throw new UnsupportedOperationException("This map is unmodifiable"); - } + @Override + public void registeredDomain(String registeredDomain) { + this.registeredDomain = registeredDomain; + } - @Override - public void clear() { - throw new UnsupportedOperationException("This map is unmodifiable"); - } + @Override + public void topLevelDomain(String topLevelDomain) { + this.topLevelDomain = topLevelDomain; + } - @Override - public Set keySet() { - return OUTPUT_FIELD_KEYS; - } + @Override + public void subdomain(String subdomain) { + this.subdomain = subdomain; + } - @Override - public Collection values() { - throw new UnsupportedOperationException("This map does not support iteration over values"); + public DomainInfo build() { + return new DomainInfo(domain, registeredDomain, topLevelDomain, subdomain); + } } + } - @Override - public Set> entrySet() { - throw new UnsupportedOperationException("This map does not support iteration over entries"); - } + public static DomainInfo parseRegisteredDomainInfo(@Nullable final String fqdn) { + DomainInfo.Factory factory = new DomainInfo.Factory(); + boolean infoFound = parseRegisteredDomainInfo(fqdn, factory); + return infoFound ? factory.build() : null; } } diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java index 07a38921b514a..14ef805348536 100644 --- a/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/UriParts.java @@ -16,6 +16,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; public class UriParts { @@ -31,41 +32,47 @@ public class UriParts { private static final String USERNAME = "username"; private static final String PASSWORD = "password"; - private static final Map> URI_PARTS_TYPES = Map.ofEntries( - Map.entry(DOMAIN, String.class), - Map.entry(FRAGMENT, String.class), - Map.entry(PATH, String.class), - Map.entry(PORT, Integer.class), - Map.entry(QUERY, String.class), - Map.entry(SCHEME, String.class), - Map.entry(USER_INFO, String.class), - Map.entry(EXTENSION, String.class), - Map.entry(USERNAME, String.class), - Map.entry(PASSWORD, String.class) - ); - - public static Map> getUriPartsTypes() { + private static final LinkedHashMap> URI_PARTS_TYPES; + + static { + URI_PARTS_TYPES = new LinkedHashMap<>(); + URI_PARTS_TYPES.putLast(DOMAIN, String.class); + URI_PARTS_TYPES.putLast(FRAGMENT, String.class); + URI_PARTS_TYPES.putLast(PATH, String.class); + URI_PARTS_TYPES.putLast(PORT, Integer.class); + URI_PARTS_TYPES.putLast(QUERY, String.class); + URI_PARTS_TYPES.putLast(SCHEME, String.class); + URI_PARTS_TYPES.putLast(EXTENSION, String.class); + URI_PARTS_TYPES.putLast(USER_INFO, String.class); + URI_PARTS_TYPES.putLast(USERNAME, String.class); + URI_PARTS_TYPES.putLast(PASSWORD, String.class); + } + + public static LinkedHashMap> getUriPartsTypes() { return URI_PARTS_TYPES; } public static Map parse(String uriString) { + final var uriParts = new UriPartsMapCollector(); + parse(uriString, uriParts); + return uriParts; + } + + @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") + public static void parse(final String uriString, final UriPartsCollector uriPartsCollector) { URI uri = null; - URL url = null; + URL fallbackUrl = null; try { uri = new URI(uriString); } catch (URISyntaxException e) { try { - url = new URL(uriString); + // noinspection deprecation + fallbackUrl = new URL(uriString); } catch (MalformedURLException e2) { throw new IllegalArgumentException("unable to parse URI [" + uriString + "]"); } } - return getUriParts(uri, url); - } - @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") - private static Map getUriParts(URI uri, URL fallbackUrl) { - var uriParts = new HashMap(); String domain; String fragment; String path; @@ -95,12 +102,12 @@ private static Map getUriParts(URI uri, URL fallbackUrl) { throw new IllegalArgumentException("at least one argument must be non-null"); } - uriParts.put(DOMAIN, domain); + uriPartsCollector.domain(domain); if (fragment != null) { - uriParts.put(FRAGMENT, fragment); + uriPartsCollector.fragment(fragment); } if (path != null) { - uriParts.put(PATH, path); + uriPartsCollector.path(path); // To avoid any issues with extracting the extension from a path that contains a dot, we explicitly extract the extension // from the last segment in the path. var lastSegmentIndex = path.lastIndexOf('/'); @@ -109,26 +116,125 @@ private static Map getUriParts(URI uri, URL fallbackUrl) { int periodIndex = lastSegment.lastIndexOf('.'); if (periodIndex >= 0) { // Don't include the dot in the extension field. - uriParts.put(EXTENSION, lastSegment.substring(periodIndex + 1)); + uriPartsCollector.extension(lastSegment.substring(periodIndex + 1)); } } } if (port != -1) { - uriParts.put(PORT, port); + uriPartsCollector.port(port); } if (query != null) { - uriParts.put(QUERY, query); + uriPartsCollector.query(query); } - uriParts.put(SCHEME, scheme); + uriPartsCollector.scheme(scheme); if (userInfo != null) { - uriParts.put(USER_INFO, userInfo); + uriPartsCollector.userInfo(userInfo); if (userInfo.contains(":")) { int colonIndex = userInfo.indexOf(':'); - uriParts.put(USERNAME, userInfo.substring(0, colonIndex)); - uriParts.put(PASSWORD, colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); + uriPartsCollector.username(userInfo.substring(0, colonIndex)); + uriPartsCollector.password(colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); } } + } - return uriParts; + /** + * A dedicated collector for URI parts. Implementation can be specific to the use case, for example, it can avoid map instance + * allocation and primitive value boxing by writing directly to the collecting data structure. + */ + public interface UriPartsCollector { + void domain(String domain); + + void fragment(String fragment); + + void path(String path); + + void extension(String extension); + + void port(int port); + + void query(String query); + + void scheme(String scheme); + + void userInfo(String userInfo); + + void username(String username); + + void password(String password); + } + + /** + * A default implementation of {@link UriPartsCollector} that writes to a {@link Map}. + */ + public static final class UriPartsMapCollector extends HashMap implements UriPartsCollector { + @Override + public void domain(String domain) { + if (domain != null) { + put(DOMAIN, domain); + } + } + + @Override + public void fragment(String fragment) { + if (fragment != null) { + put(FRAGMENT, fragment); + } + } + + @Override + public void path(String path) { + if (path != null) { + put(PATH, path); + } + } + + @Override + public void extension(String extension) { + if (extension != null) { + put(EXTENSION, extension); + } + } + + @Override + public void port(int port) { + if (port >= 0) { + put(PORT, port); + } + } + + @Override + public void query(String query) { + if (query != null) { + put(QUERY, query); + } + } + + @Override + public void scheme(String scheme) { + if (scheme != null) { + put(SCHEME, scheme); + } + } + + @Override + public void userInfo(String userInfo) { + if (userInfo != null) { + put(USER_INFO, userInfo); + } + } + + @Override + public void username(String username) { + if (username != null) { + put(USERNAME, username); + } + } + + @Override + public void password(String password) { + if (password != null) { + put(PASSWORD, password); + } + } } } diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java index e2e643882ced0..dbb5d6effa4b1 100644 --- a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.web.RegisteredDomain.getRegisteredDomainInfo; +import static org.elasticsearch.web.RegisteredDomain.parseRegisteredDomainInfo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -19,38 +19,41 @@ public class RegisteredDomainTests extends ESTestCase { public void testGetRegisteredDomain() { assertThat( - getRegisteredDomainInfo("www.google.com"), + parseRegisteredDomainInfo("www.google.com"), is(new RegisteredDomain.DomainInfo("www.google.com", "google.com", "com", "www")) ); - assertThat(getRegisteredDomainInfo("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); - assertThat(getRegisteredDomainInfo(null), nullValue()); - assertThat(getRegisteredDomainInfo(""), nullValue()); - assertThat(getRegisteredDomainInfo(" "), nullValue()); - assertThat(getRegisteredDomainInfo("."), nullValue()); - assertThat(getRegisteredDomainInfo("$"), nullValue()); - assertThat(getRegisteredDomainInfo("foo.bar.baz"), nullValue()); + assertThat(parseRegisteredDomainInfo("google.com"), is(new RegisteredDomain.DomainInfo("google.com", "google.com", "com", null))); + assertThat(parseRegisteredDomainInfo(null), nullValue()); + assertThat(parseRegisteredDomainInfo(""), nullValue()); + assertThat(parseRegisteredDomainInfo(" "), nullValue()); + assertThat(parseRegisteredDomainInfo("."), nullValue()); + assertThat(parseRegisteredDomainInfo("$"), nullValue()); + assertThat(parseRegisteredDomainInfo("foo.bar.baz"), nullValue()); assertThat( - getRegisteredDomainInfo("www.books.amazon.co.uk"), + parseRegisteredDomainInfo("www.books.amazon.co.uk"), is(new RegisteredDomain.DomainInfo("www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books")) ); // Verify "com" is returned as the eTLD, for that FQDN or subdomain - assertThat(getRegisteredDomainInfo("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); - assertThat(getRegisteredDomainInfo("example.com"), is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null))); + assertThat(parseRegisteredDomainInfo("com"), is(new RegisteredDomain.DomainInfo("com", null, "com", null))); assertThat( - getRegisteredDomainInfo("googleapis.com"), + parseRegisteredDomainInfo("example.com"), + is(new RegisteredDomain.DomainInfo("example.com", "example.com", "com", null)) + ); + assertThat( + parseRegisteredDomainInfo("googleapis.com"), is(new RegisteredDomain.DomainInfo("googleapis.com", "googleapis.com", "com", null)) ); assertThat( - getRegisteredDomainInfo("content-autofill.googleapis.com"), + parseRegisteredDomainInfo("content-autofill.googleapis.com"), is(new RegisteredDomain.DomainInfo("content-autofill.googleapis.com", "googleapis.com", "com", "content-autofill")) ); // Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain assertThat( - getRegisteredDomainInfo("global.ssl.fastly.net"), + parseRegisteredDomainInfo("global.ssl.fastly.net"), is(new RegisteredDomain.DomainInfo("global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null)) ); assertThat( - getRegisteredDomainInfo("1.www.global.ssl.fastly.net"), + parseRegisteredDomainInfo("1.www.global.ssl.fastly.net"), is(new RegisteredDomain.DomainInfo("1.www.global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", "1.www")) ); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index 13b7e140e37f3..4d438e6941bd8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -48,34 +48,16 @@ public boolean getIgnoreMissing() { @Override public IngestDocument execute(IngestDocument document) throws Exception { final String fqdn = document.getFieldValue(field, String.class, ignoreMissing); - final Map info = RegisteredDomain.getRegisteredDomainInfo(fqdn); - if (info == null) { - if (ignoreMissing) { - return document; - } else { - throw new IllegalArgumentException("unable to set domain information for document"); - } - } String fieldPrefix = targetField; if (fieldPrefix.isEmpty() == false) { fieldPrefix += "."; } - - String domain = info.get(RegisteredDomain.DOMAIN); - if (domain != null) { - document.setFieldValue(fieldPrefix + RegisteredDomain.DOMAIN, domain); - } - String registeredDomain = info.get(RegisteredDomain.REGISTERED_DOMAIN); - if (registeredDomain != null) { - document.setFieldValue(fieldPrefix + RegisteredDomain.REGISTERED_DOMAIN, registeredDomain); - } - String eTLD = info.get(RegisteredDomain.ETLD); - if (eTLD != null) { - document.setFieldValue(fieldPrefix + RegisteredDomain.ETLD, eTLD); - } - String subdomain = info.get(RegisteredDomain.SUBDOMAIN); - if (subdomain != null) { - document.setFieldValue(fieldPrefix + RegisteredDomain.SUBDOMAIN, subdomain); + boolean infoFound = RegisteredDomain.parseRegisteredDomainInfo( + fqdn, + new IngestDocumentRegisteredDomainInfoCollector(document, fieldPrefix) + ); + if (infoFound == false && ignoreMissing == false) { + throw new IllegalArgumentException("unable to set domain information for document"); } return document; } @@ -104,4 +86,34 @@ public RegisteredDomainProcessor create( return new RegisteredDomainProcessor(tag, description, field, targetField, ignoreMissing); } } + + private static class IngestDocumentRegisteredDomainInfoCollector implements RegisteredDomain.RegisteredDomainInfoCollector { + private final IngestDocument document; + private final String fieldPrefix; + + IngestDocumentRegisteredDomainInfoCollector(IngestDocument document, String fieldPrefix) { + this.document = document; + this.fieldPrefix = fieldPrefix; + } + + @Override + public void domain(String domain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.DOMAIN, domain); + } + + @Override + public void registeredDomain(String registeredDomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.REGISTERED_DOMAIN, registeredDomain); + } + + @Override + public void topLevelDomain(String topLevelDomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.ETLD, topLevelDomain); + } + + @Override + public void subdomain(String subdomain) { + document.setFieldValue(fieldPrefix + RegisteredDomain.SUBDOMAIN, subdomain); + } + } } From 6d78b71400b1a7919a427c8b9f01719ee1953d0a Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:08:51 +0200 Subject: [PATCH 09/11] Applying review info --- libs/web-utils/build.gradle | 1 - .../elasticsearch/web/RegisteredDomain.java | 20 +++++++++++++++++-- .../common/RegisteredDomainProcessor.java | 2 +- .../390_registered_domain_processor.yml | 1 - 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle index 1efe7c32e09a5..f6ff315e75f22 100644 --- a/libs/web-utils/build.gradle +++ b/libs/web-utils/build.gradle @@ -23,7 +23,6 @@ tasks.named('forbiddenApisMain').configure { tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( - //commons-logging 'org.apache.commons.codec.binary.Base64', 'org.apache.commons.logging.Log', 'org.apache.commons.logging.LogFactory', diff --git a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java index 9bd3b98df0480..70975895428d4 100644 --- a/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java +++ b/libs/web-utils/src/main/java/org/elasticsearch/web/RegisteredDomain.java @@ -15,11 +15,27 @@ import java.util.LinkedHashMap; +/** + * Utility class for parsing fully qualified domain names (FQDNs) into their constituent parts: + * domain, registered domain, top-level domain (eTLD), and subdomain. + *

+ * This class uses the public suffix list to accurately determine domain boundaries. + * For example, given "www.example.co.uk": + *

    + *
  • domain: www.example.co.uk
    • + *
  • registered_domain: example.co.uk
    • + *
  • top_level_domain: co.uk
    • + *
  • subdomain: www
    • + *
+ * + * @see Public Suffix List + * @see eTLD (effective Top-Level Domain) + */ public class RegisteredDomain { public static final String DOMAIN = "domain"; public static final String REGISTERED_DOMAIN = "registered_domain"; - public static final String ETLD = "top_level_domain"; + public static final String eTLD = "top_level_domain"; public static final String SUBDOMAIN = "subdomain"; public static final LinkedHashMap> REGISTERED_DOMAIN_INFO_FIELDS; @@ -28,7 +44,7 @@ public class RegisteredDomain { REGISTERED_DOMAIN_INFO_FIELDS = new LinkedHashMap<>(); REGISTERED_DOMAIN_INFO_FIELDS.putLast(DOMAIN, String.class); REGISTERED_DOMAIN_INFO_FIELDS.putLast(REGISTERED_DOMAIN, String.class); - REGISTERED_DOMAIN_INFO_FIELDS.putLast(ETLD, String.class); + REGISTERED_DOMAIN_INFO_FIELDS.putLast(eTLD, String.class); REGISTERED_DOMAIN_INFO_FIELDS.putLast(SUBDOMAIN, String.class); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java index 4d438e6941bd8..f67b09955291a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -108,7 +108,7 @@ public void registeredDomain(String registeredDomain) { @Override public void topLevelDomain(String topLevelDomain) { - document.setFieldValue(fieldPrefix + RegisteredDomain.ETLD, topLevelDomain); + document.setFieldValue(fieldPrefix + RegisteredDomain.eTLD, topLevelDomain); } @Override diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml index 6b38f8907a4ed..1bac618e7b874 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/390_registered_domain_processor.yml @@ -35,7 +35,6 @@ setup: body: settings: number_of_shards: 1 - number_of_replicas: 0 mappings: properties: domain_input: { type: keyword } From 30ce24cdc4e96cd28763d4a0bf6389926bab8362 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 5 Feb 2026 13:13:55 +0200 Subject: [PATCH 10/11] Removing module exclusion in the test-framework dependency declaration --- libs/web-utils/build.gradle | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/web-utils/build.gradle b/libs/web-utils/build.gradle index f6ff315e75f22..adad51f4f9ceb 100644 --- a/libs/web-utils/build.gradle +++ b/libs/web-utils/build.gradle @@ -12,9 +12,7 @@ dependencies { implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" - testImplementation(project(":test:framework")) { - exclude group: 'org.elasticsearch', module: 'web-utils' - } + testImplementation(project(":test:framework")) } tasks.named('forbiddenApisMain').configure { From 0d6968e0568c773c211e2b532317e8bf765fde3f Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 8 Feb 2026 16:18:00 +0200 Subject: [PATCH 11/11] Disabling Entitlements check in test --- .../org/elasticsearch/web/RegisteredDomainTests.java | 1 + .../org/elasticsearch/bootstrap/TestScopeResolver.java | 10 +--------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java index dbb5d6effa4b1..2016bfd6da2cd 100644 --- a/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java +++ b/libs/web-utils/src/test/java/org/elasticsearch/web/RegisteredDomainTests.java @@ -15,6 +15,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +@ESTestCase.WithoutEntitlements public class RegisteredDomainTests extends ESTestCase { public void testGetRegisteredDomain() { diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java index 765261fc664e1..91662f3f35773 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/TestScopeResolver.java @@ -82,15 +82,7 @@ PolicyScope getScope(Class callerClass) { if (callerClass.getPackageName().startsWith("org.bouncycastle")) { scope = new PolicyScope(PLUGIN, "security", ALL_UNNAMED); logger.debug("Assuming bouncycastle is part of the security plugin"); - } else if (callerClass.getPackageName().startsWith("org.apache.httpcomponents") - || callerClass.getPackageName().startsWith("org.apache.http.conn")) { - String moduleName = callerClass.getModule().getName(); - if (moduleName == null) { - moduleName = ALL_UNNAMED; - } - scope = new PolicyScope(PLUGIN, "ingest-common/esql", moduleName); - logger.debug("Assuming httpclient is related to the ingest-common and esql plugins for tests"); - } + } } if (scope == null) { logger.warn("Cannot identify a scope for class [{}], location [{}]", callerClass.getName(), location);