Remove cap on Java version used by forbidden APIs

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adding ScriptedAvg class to painless spi to allowlist usage from plugins ([#19006](https://github.com/opensearch-project/OpenSearch/pull/19006))
 - Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
 - Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
+- Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.8'
+  api 'de.thetaphi:forbiddenapis:3.9'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java

@@ -40,7 +40,6 @@
 import org.opensearch.gradle.ExportOpenSearchBuildResourcesTask;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.util.GradleUtils;
-import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
@@ -53,6 +52,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin {
     @Override
@@ -89,10 +89,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.setClasspath(project.files(sourceSet.getRuntimeClasspath()).plus(sourceSet.getCompileClasspath()));
 
             t.setTargetCompatibility(BuildParams.getRuntimeJavaVersion().getMajorVersion());
-            if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) > 0) {
-                // TODO: forbidden apis does not yet support java 15, rethink using runtime version
-                t.setTargetCompatibility(JavaVersion.VERSION_14.getMajorVersion());
-            }
             t.setBundledSignatures(new HashSet<>(Arrays.asList("jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out")));
             t.setSignaturesFiles(
                 project.files(
@@ -140,6 +136,18 @@ public Void call(Object... names) {
                     return null;
                 }
             });
+            // Use of the deprecated security manager APIs are pervasive so set them to warn
+            // globally for all projects. Replacements for (most of) these APIs are available
+            // so usages can move to the non-deprecated variants to avoid the warnings.
+            t.setSignaturesWithSeverityWarn(
+                Set.of(
+                    "java.security.AccessController",
+                    "java.security.AccessControlContext",
+                    "java.lang.System#getSecurityManager()",
+                    "java.lang.SecurityManager",
+                    "java.security.Policy"
+                )
+            );
         });
         TaskProvider<Task> forbiddenApis = project.getTasks().named("forbiddenApis");
         forbiddenApis.configure(t -> t.setGroup(""));

distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java

@@ -399,7 +399,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf
     @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
     boolean urlExists(Terminal terminal, String urlString) throws IOException {
         terminal.println(VERBOSE, "Checking if url exists: " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         assert "https".equals(url.getProtocol()) : "Use of https protocol is required";
         HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -427,7 +427,7 @@ private List<String> checkMisspelledPlugin(String pluginId) {
     @SuppressForbidden(reason = "We use getInputStream to download plugins")
     Path downloadZip(Terminal terminal, String urlString, Path tmpDir, boolean isBatch) throws IOException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
         URLConnection urlConnection = url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -684,7 +684,7 @@ InputStream getPublicKey() {
      */
     // pkg private for tests
     URL openUrl(String urlString) throws IOException {
-        URL checksumUrl = new URL(urlString);
+        URL checksumUrl = URI.create(urlString).toURL();
         HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection();
         if (connection.getResponseCode() == 404) {
             return null;

distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java

@@ -526,7 +526,7 @@ public void testSpaceInUrl() throws Exception {
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPluginUrl("fake", pluginDir);
         Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
-        try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip))) {
+        try (InputStream in = FileSystemUtils.openFileURLStream(URI.create(pluginZip).toURL())) {
             Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
         }
         installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
@@ -536,8 +536,8 @@ public void testSpaceInUrl() throws Exception {
     public void testMalformedUrlNotMaven() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         // has two colons, so it appears similar to maven coordinates
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> installPlugin("://host:1234", env.v1()));
+        assertThat(e.getMessage(), startsWith("Expected scheme name"));
     }
 
     public void testFileNotMaven() throws Exception {

libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java

@@ -40,6 +40,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.ByteBuffer;
@@ -132,21 +133,21 @@ public void testIsHidden() {
     }
 
     public void testOpenFileURLStream() throws IOException {
-        URL urlWithWrongProtocol = new URL("http://www.google.com");
+        URL urlWithWrongProtocol = URI.create("http://www.google.com").toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithWrongProtocol)) {
             fail("Should throw IllegalArgumentException due to invalid protocol");
         } catch (IllegalArgumentException e) {
             assertEquals("Invalid protocol [http], must be [file] or [jar]", e.getMessage());
         }
 
-        URL urlWithHost = new URL("file", "localhost", txtFile.toString());
+        URL urlWithHost = URI.create("file://localhost/" + txtFile.toString()).toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithHost)) {
             fail("Should throw IllegalArgumentException due to host");
         } catch (IllegalArgumentException e) {
             assertEquals("URL cannot have host. Found: [localhost]", e.getMessage());
         }
 
-        URL urlWithPort = new URL("file", "", 80, txtFile.toString());
+        URL urlWithPort = URI.create("file://:80/" + txtFile.toString()).toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithPort)) {
             fail("Should throw IllegalArgumentException due to port");
         } catch (IllegalArgumentException e) {

libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java

@@ -77,12 +77,12 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) {
     private void assertHasTrustedIssuer(X509ExtendedTrustManager trustManager, String name) {
         final String lowerName = name.toLowerCase(Locale.ROOT);
         final Optional<X509Certificate> ca = Stream.of(trustManager.getAcceptedIssuers())
-            .filter(cert -> cert.getSubjectDN().getName().toLowerCase(Locale.ROOT).contains(lowerName))
+            .filter(cert -> cert.getSubjectX500Principal().getName().toLowerCase(Locale.ROOT).contains(lowerName))
             .findAny();
         if (ca.isPresent() == false) {
             logger.info("Failed to find issuer [{}] in trust manager, but did find ...", lowerName);
             for (X509Certificate cert : trustManager.getAcceptedIssuers()) {
-                logger.info(" - {}", cert.getSubjectDN().getName().replaceFirst("^\\w+=([^,]+),.*", "$1"));
+                logger.info(" - {}", cert.getSubjectX500Principal().getName().replaceFirst("^\\w+=([^,]+),.*", "$1"));
             }
             Assert.fail("Cannot find trusted issuer with name [" + name + "].");
         }

libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java

@@ -154,8 +154,8 @@ private void assertCertificateAndKey(PemKeyConfig keyConfig, String expectedDN)
         assertThat(chain, notNullValue());
         assertThat(chain, arrayWithSize(1));
         final X509Certificate certificate = chain[0];
-        assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1"));
-        assertThat(certificate.getSubjectDN().getName(), is(expectedDN));
+        assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1"));
+        assertThat(certificate.getSubjectX500Principal().getName(), is(expectedDN));
         assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2));
         assertThat(
             certificate.getSubjectAlternativeNames(),

libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java

@@ -146,7 +146,7 @@ private void assertCertificateChain(PemTrustConfig trustConfig, String... caName
         final X509ExtendedTrustManager trustManager = trustConfig.createTrustManager();
         final X509Certificate[] issuers = trustManager.getAcceptedIssuers();
         final Set<String> issuerNames = Stream.of(issuers)
-            .map(X509Certificate::getSubjectDN)
+            .map(X509Certificate::getSubjectX500Principal)
             .map(Principal::getName)
             .collect(Collectors.toSet());
 

libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java

@@ -183,8 +183,8 @@ private void assertKeysLoaded(StoreKeyConfig keyConfig, String... names) throws
             assertThat(chain, notNullValue());
             assertThat(chain, arrayWithSize(1));
             final X509Certificate certificate = chain[0];
-            assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1"));
-            assertThat(certificate.getSubjectDN().getName(), is("CN=" + name));
+            assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1"));
+            assertThat(certificate.getSubjectX500Principal().getName(), is("CN=" + name));
             assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2));
             assertThat(
                 certificate.getSubjectAlternativeNames(),

libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreTrustConfigTests.java

@@ -140,7 +140,7 @@ private void assertCertificateChain(StoreTrustConfig trustConfig, String... caNa
         final X509ExtendedTrustManager trustManager = trustConfig.createTrustManager();
         final X509Certificate[] issuers = trustManager.getAcceptedIssuers();
         final Set<String> issuerNames = Stream.of(issuers)
-            .map(X509Certificate::getSubjectDN)
+            .map(X509Certificate::getSubjectX500Principal)
             .map(Principal::getName)
             .collect(Collectors.toSet());
 

modules/ingest-common/src/test/java/org/opensearch/ingest/common/DateProcessorTests.java

@@ -46,12 +46,14 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.IllformedLocaleException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
 
 public class DateProcessorTests extends OpenSearchTestCase {
 
@@ -315,7 +317,7 @@ public void testInvalidLocale() {
             () -> processor.execute(RandomDocumentPicks.randomIngestDocument(random(), document))
         );
         assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
-        assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid"));
+        assertThat(e.getCause(), instanceOf(IllformedLocaleException.class));
     }
 
     public void testOutputFormat() {

modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java

@@ -50,7 +50,7 @@
 
 import java.lang.reflect.Method;
 import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.URI;
 import java.security.CodeSource;
 import java.security.SecureClassLoader;
 import java.security.cert.Certificate;
@@ -77,7 +77,7 @@ final class Compiler {
     static {
         try {
             // Setup the code privileges.
-            CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null);
+            CODESOURCE = new CodeSource(URI.create("file:" + BootstrapInfo.UNTRUSTED_CODEBASE).toURL(), (Certificate[]) null);
         } catch (MalformedURLException impossible) {
             throw new RuntimeException(impossible);
         }

modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java

@@ -57,7 +57,7 @@
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.URI;
 import java.security.AccessController;
 import java.security.CodeSource;
 import java.security.PrivilegedAction;
@@ -120,7 +120,7 @@ Class<?> defineBridge(String name, byte[] bytes) {
 
     static {
         try {
-            CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null);
+            CODESOURCE = new CodeSource(URI.create("file:" + BootstrapInfo.UNTRUSTED_CODEBASE).toURL(), (Certificate[]) null);
         } catch (MalformedURLException mue) {
             throw new RuntimeException(mue);
         }

modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java

@@ -43,6 +43,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
 import java.security.AccessController;
@@ -136,9 +137,11 @@ public DeleteResult delete() {
     @Override
     public InputStream readBlob(String name) throws IOException {
         try {
-            return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes());
+            return new BufferedInputStream(getInputStream(this.path.toURI().resolve(name).toURL()), blobStore.bufferSizeInBytes());
         } catch (FileNotFoundException fnfe) {
             throw new NoSuchFileException("[" + name + "] blob not found");
+        } catch (URISyntaxException e) {
+            throw new IOException(e);
         }
     }
 

modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java

@@ -41,6 +41,7 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 
 import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.net.URL;
 
 /**
@@ -97,7 +98,7 @@ public int bufferSizeInBytes() {
     public BlobContainer blobContainer(BlobPath path) {
         try {
             return new URLBlobContainer(this, path, buildPath(path));
-        } catch (MalformedURLException ex) {
+        } catch (MalformedURLException | URISyntaxException ex) {
             throw new BlobStoreException("malformed URL " + path, ex);
         }
     }
@@ -113,17 +114,15 @@ public void close() {
      * @param path relative path
      * @return Base URL + path
      */
-    private URL buildPath(BlobPath path) throws MalformedURLException {
+    private URL buildPath(BlobPath path) throws MalformedURLException, URISyntaxException {
         String[] paths = path.toArray();
         if (paths.length == 0) {
             return path();
         }
-        URL blobPath = new URL(this.path, paths[0] + "/");
-        if (paths.length > 1) {
-            for (int i = 1; i < paths.length; i++) {
-                blobPath = new URL(blobPath, paths[i] + "/");
-            }
+        var uri = this.path.toURI();
+        for (String pathElement : paths) {
+            uri = uri.resolve(pathElement + "/");
         }
-        return blobPath;
+        return uri.toURL();
     }
 }

modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java

@@ -50,6 +50,7 @@
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Arrays;
@@ -85,10 +86,10 @@ public class URLRepository extends BlobStoreRepository {
         Property.NodeScope
     );
 
-    public static final Setting<URL> URL_SETTING = new Setting<>("url", "http:", URLRepository::parseURL, Property.NodeScope);
+    public static final Setting<URL> URL_SETTING = new Setting<>("url", "http://?", URLRepository::parseURL, Property.NodeScope);
     public static final Setting<URL> REPOSITORIES_URL_SETTING = new Setting<>(
         "repositories.url.url",
-        (s) -> s.get("repositories.uri.url", "http:"),
+        (s) -> s.get("repositories.uri.url", "http://?"),
         URLRepository::parseURL,
         Property.NodeScope
     );
@@ -194,7 +195,7 @@ public boolean isReadOnly() {
 
     private static URL parseURL(String s) {
         try {
-            return new URL(s);
+            return URI.create(s).toURL();
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException("Unable to parse URL repository setting", e);
         }

modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -55,7 +55,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.URI;
-import java.net.URL;
 import java.util.List;
 import java.util.Map;
 
@@ -120,7 +119,7 @@ public void registerRepositories() throws IOException {
         List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
         for (String allowedUrl : allowedUrls) {
             try {
-                InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
+                InetAddress inetAddress = InetAddress.getByName(URI.create(allowedUrl).getHost());
                 if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
                     Request createUrlRepositoryRequest = new Request("PUT", "/_snapshot/repository-url");
                     createUrlRepositoryRequest.setEntity(buildRepositorySettings("url", Settings.builder().put("url", allowedUrl).build()));

plugins/examples/rest-handler/src/javaRestTest/java/org/opensearch/example/resthandler/ExampleFixtureIT.java

@@ -40,6 +40,7 @@
 import java.io.OutputStreamWriter;
 import java.net.InetAddress;
 import java.net.Socket;
+import java.net.URI;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -53,7 +54,7 @@ public void testExample() throws Exception {
         final String externalAddress = System.getProperty("external.address");
         assertNotNull("External address must not be null", externalAddress);
 
-        final URL url = new URL("http://" + externalAddress);
+        final URL url = URI.create("http://" + externalAddress).toURL();
         final InetAddress address = InetAddress.getByName(url.getHost());
         try (
             Socket socket = new Socket(address, url.getPort());

qa/evil-tests/src/test/java/org/opensearch/bootstrap/SystemCallFilterTests.java

@@ -39,7 +39,7 @@
 public class SystemCallFilterTests extends OpenSearchTestCase {
 
     /** command to try to run in tests */
-    static final String EXECUTABLE = Constants.WINDOWS ? "calc" : "ls";
+    static final String[] EXECUTABLE = new String[] { Constants.WINDOWS ? "calc" : "ls" };
 
     @SuppressWarnings("removal")
     @Override