WIP on Secure GRPC Transport w/ integ test

build.gradle

@@ -494,6 +494,9 @@ configurations {
             force "net.bytebuddy:byte-buddy:1.17.6"
             force "org.ow2.asm:asm:9.8"
             force "com.google.j2objc:j2objc-annotations:3.0.0"
+            force "com.google.protobuf:protobuf-java:3.25.5"
+            force "io.perfmark:perfmark-api:0.27.0"
+            force "com.google.guava:failureaccess:1.0.3"
         }
     }
 
@@ -533,6 +536,11 @@ allprojects {
         integrationTestImplementation("org.opensearch.plugin:reindex-client:${opensearch_version}"){
             exclude(group: 'org.slf4j', module: 'slf4j-api')
         }
+        integrationTestImplementation "org.opensearch:protobufs:0.3.0"
+        integrationTestImplementation "io.grpc:grpc-protobuf:${versions.grpc}"
+        integrationTestImplementation "io.grpc:grpc-stub:${versions.grpc}"
+        integrationTestImplementation "io.grpc:grpc-netty-shaded:${versions.grpc}"
+        integrationTestImplementation "org.opensearch.plugin:transport-grpc:${opensearch_version}"
         integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
         integrationTestImplementation 'commons-io:commons-io:2.19.0'
         integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"

sample-resource-plugin/build.gradle

@@ -67,6 +67,8 @@ configurations.all {
         force 'org.hamcrest:hamcrest:2.2'
         force 'org.mockito:mockito-core:5.18.0'
         force 'org.slf4j:slf4j-api:1.7.36'
+        force 'com.google.errorprone:error_prone_annotations:2.36.0'
+        force 'com.google.guava:guava:33.4.8-jre'
     }
 }
 

src/integrationTest/java/org/opensearch/security/api/CertificatesRestApiIntegrationTest.java

@@ -14,16 +14,15 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 import java.util.StringJoiner;
 import java.util.stream.Collectors;
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.fasterxml.jackson.databind.JsonNode;
 import org.junit.Test;
 
+import org.opensearch.common.CheckedConsumer;
 import org.opensearch.security.dlic.rest.api.Endpoint;
 import org.opensearch.security.ssl.config.CertType;
 import org.opensearch.test.framework.TestSecurityConfig;
@@ -37,12 +36,12 @@
 import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX;
 import static org.opensearch.security.dlic.rest.api.RestApiAdminPrivilegesEvaluator.CERTS_INFO_ACTION;
 import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED;
+import static junit.framework.TestCase.fail;
 
 public class CertificatesRestApiIntegrationTest extends AbstractApiIntegrationTest {
-
     final static String REST_API_ADMIN_SSL_INFO = "rest-api-admin-ssl-info";
-
     final static String REGULAR_USER = "regular_user";
+    final static String ROOT_CA = "Root CA";
 
     static {
         testSecurityConfig.roles(
@@ -89,13 +88,17 @@ public void forbiddenForAdminUser() throws Exception {
 
     @Test
     public void availableForTlsAdmin() throws Exception {
-        withUser(ADMIN_USER_NAME, localCluster.getAdminCertificate(), this::verifySSLCertsInfo);
+        withUser(
+            ADMIN_USER_NAME,
+            localCluster.getAdminCertificate(),
+            verifySSLCertsInfo(List.of(CertType.HTTP, CertType.TRANSPORT, CertType.TRANSPORT_CLIENT))
+        );
     }
 
     @Test
     public void availableForRestAdmin() throws Exception {
-        withUser(REST_ADMIN_USER, this::verifySSLCertsInfo);
-        withUser(REST_API_ADMIN_SSL_INFO, this::verifySSLCertsInfo);
+        withUser(REST_ADMIN_USER, verifySSLCertsInfo(List.of(CertType.HTTP, CertType.TRANSPORT, CertType.TRANSPORT_CLIENT)));
+        withUser(REST_API_ADMIN_SSL_INFO, verifySSLCertsInfo(List.of(CertType.HTTP, CertType.TRANSPORT, CertType.TRANSPORT_CLIENT)));
     }
 
     @Test
@@ -107,72 +110,78 @@ private void verifyTimeoutRequest(final TestRestClient client) throws Exception
         ok(() -> client.get(sslCertsPath() + "?timeout=0"));
     }
 
-    private void verifySSLCertsInfo(final TestRestClient client) throws Exception {
-        assertSSLCertsInfo(localCluster.nodes(), CertType.TYPES, ok(() -> client.get(sslCertsPath())));
-        if (localCluster.nodes().size() > 1) {
-            final var randomNodes = randomNodes();
-            final var nodeIds = randomNodes.stream().map(n -> n.esNode().getNodeEnvironment().nodeId()).collect(Collectors.joining(","));
-            assertSSLCertsInfo(randomNodes, CertType.TYPES, ok(() -> client.get(sslCertsPath(nodeIds))));
-        }
-        final var randomCertType = randomFrom(List.copyOf(CertType.TYPES));
-        assertSSLCertsInfo(
-            localCluster.nodes(),
-            Set.of(randomCertType),
-            ok(() -> client.get(String.format("%s?cert_type=%s", sslCertsPath(), randomCertType)))
-        );
-
+    private CheckedConsumer<TestRestClient, Exception> verifySSLCertsInfo(List<CertType> expectCerts) {
+        return testRestClient -> {
+            try {
+                assertSSLCertsInfo(localCluster.nodes(), expectCerts, ok(() -> testRestClient.get(sslCertsPath())));
+                if (localCluster.nodes().size() > 1) {
+                    final var randomNodes = randomNodes();
+                    final var nodeIds = randomNodes.stream()
+                        .map(n -> n.esNode().getNodeEnvironment().nodeId())
+                        .collect(Collectors.joining(","));
+                    assertSSLCertsInfo(randomNodes, expectCerts, ok(() -> testRestClient.get(sslCertsPath(nodeIds))));
+                }
+                final var randomCertType = randomFrom(expectCerts);
+                assertSSLCertsInfo(
+                    localCluster.nodes(),
+                    List.of(randomCertType),
+                    ok(() -> testRestClient.get(String.format("%s?cert_type=%s", sslCertsPath(), randomCertType)))
+                );
+            } catch (Exception e) {
+                fail("Verify SSLCerts info failed with exception: " + e.getMessage());
+            }
+        };
     }
 
     private void assertSSLCertsInfo(
         final List<LocalOpenSearchCluster.Node> expectedNode,
-        final Set<String> expectedCertTypes,
+        final List<CertType> expectedCertTypes,
         final TestRestClient.HttpResponse response
     ) {
         final var body = response.bodyAsJsonNode();
         final var prettyStringBody = body.toPrettyString();
-
         final var _nodes = body.get("_nodes");
         assertThat(prettyStringBody, _nodes.get("total").asInt(), is(expectedNode.size()));
         assertThat(prettyStringBody, _nodes.get("successful").asInt(), is(expectedNode.size()));
         assertThat(prettyStringBody, _nodes.get("failed").asInt(), is(0));
         assertThat(prettyStringBody, body.get("cluster_name").asText(), is(localCluster.getClusterName()));
-
         final var nodes = body.get("nodes");
-
         for (final var n : expectedNode) {
             final var esNode = n.esNode();
             final var node = nodes.get(esNode.getNodeEnvironment().nodeId());
             assertThat(prettyStringBody, node.get("name").asText(), is(n.getNodeName()));
             assertThat(prettyStringBody, node.has("certificates"));
             final var certificates = node.get("certificates");
-            if (expectedCertTypes.contains(CertType.HTTP.name().toUpperCase(Locale.ROOT))) {
-                final var httpCertificates = certificates.get(CertType.HTTP.name().toUpperCase(Locale.ROOT));
-                assertThat(prettyStringBody, httpCertificates.isArray());
-                assertThat(prettyStringBody, httpCertificates.size(), is(1));
-                verifyCertsJson(n.nodeNumber(), httpCertificates.get(0));
-            }
-            if (expectedCertTypes.contains(CertType.TRANSPORT_CLIENT.name().toUpperCase(Locale.ROOT))) {
-                final var transportCertificates = certificates.get(CertType.TRANSPORT.name().toUpperCase(Locale.ROOT));
+            /*
+            Expect each node transport configured with root and issued cert.
+            */
+            for (CertType expectCert : expectedCertTypes) {
+                final JsonNode transportCertificates = certificates.get(expectCert.id());
                 assertThat(prettyStringBody, transportCertificates.isArray());
-                assertThat(prettyStringBody, transportCertificates.size(), is(1));
+                assertThat(prettyStringBody, transportCertificates.size(), is(2));
                 verifyCertsJson(n.nodeNumber(), transportCertificates.get(0));
+                verifyCertsJson(n.nodeNumber(), transportCertificates.get(1));
             }
         }
-
     }
 
     private void verifyCertsJson(final int nodeNumber, final JsonNode jsonNode) {
+        if (jsonNode.get("subject_dn").asText().contains(ROOT_CA)) { // handle root cert
+            assertThat(jsonNode.toPrettyString(), jsonNode.get("subject_dn").asText(), is(TestCertificates.CA_SUBJECT));
+            assertThat(jsonNode.toPrettyString(), jsonNode.get("san").asText().isEmpty());
+        } else { // handle non-root
+            assertThat(
+                jsonNode.toPrettyString(),
+                jsonNode.get("subject_dn").asText(),
+                is(String.format(TestCertificates.NODE_SUBJECT_PATTERN, nodeNumber))
+            );
+            assertThat(
+                jsonNode.toPrettyString(),
+                jsonNode.get("san").asText(),
+                containsString(String.format("node-%s.example.com", nodeNumber))
+            );
+        }
         assertThat(jsonNode.toPrettyString(), jsonNode.get("issuer_dn").asText(), is(TestCertificates.CA_SUBJECT));
-        assertThat(
-            jsonNode.toPrettyString(),
-            jsonNode.get("subject_dn").asText(),
-            is(String.format(TestCertificates.NODE_SUBJECT_PATTERN, nodeNumber))
-        );
-        assertThat(
-            jsonNode.toPrettyString(),
-            jsonNode.get("san").asText(),
-            containsString(String.format("node-%s.example.com", nodeNumber))
-        );
         assertThat(jsonNode.toPrettyString(), jsonNode.has("not_before"));
         assertThat(jsonNode.toPrettyString(), jsonNode.has("not_after"));
     }
@@ -193,5 +202,4 @@ public <T> List<T> randomSubsetOf(int size, Collection<T> collection) {
         Collections.shuffle(tempList, RandomizedContext.current().getRandom());
         return tempList.subList(0, size);
     }
-
 }

src/integrationTest/java/org/opensearch/security/grpc/GrpcTests.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security.grpc;
+
+import java.io.IOException;
+import java.util.Map;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.plugin.transport.grpc.GrpcPlugin;
+import org.opensearch.protobufs.SearchRequest;
+import org.opensearch.protobufs.SearchRequestBody;
+import org.opensearch.protobufs.SearchResponse;
+import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.test.framework.TestSecurityConfig.User;
+import org.opensearch.test.framework.certificate.TestCertificates;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestGrpcClient;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class GrpcTests {
+
+    private static final Logger log = LogManager.getLogger(GrpcTests.class);
+
+    static final User ADMIN_USER = new User("admin").roles(ALL_ACCESS);
+
+    private static final TestCertificates TEST_CERTIFICATES = new TestCertificates();
+
+    @ClassRule
+    public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE)
+        .certificates(TEST_CERTIFICATES)
+        .plugin(GrpcPlugin.class)
+        .grpc(TEST_CERTIFICATES)
+        .loadConfigurationIntoIndex(false)
+        .nodeSettings(Map.of(ConfigConstants.SECURITY_SSL_ONLY, true))
+        .sslOnly(true)
+        .build();
+
+    @Test
+    public void testSearch() throws IOException {
+        try (TestRestClient client = cluster.getRestClient()) {
+            TestRestClient.HttpResponse response = client.put("test-index");
+            response.assertStatusCode(200);
+            client.postJson("test-index/_doc/1", "{\"field\": \"value\"}");
+        }
+
+        // TODO this is SSL Only test, but eventually a test with authc should be added as well
+        TestGrpcClient client = cluster.getGrpcClient(TEST_CERTIFICATES);
+        // Create a search request
+        SearchRequestBody requestBody = SearchRequestBody.newBuilder().setFrom(0).setSize(10).build();
+
+        SearchRequest searchRequest = SearchRequest.newBuilder()
+            .addIndex("test-index")
+            .setRequestBody(requestBody)
+            .setQ("field:value")
+            .build();
+        SearchResponse response = client.search(searchRequest);
+        System.out.println("Search Response: " + response);
+    }
+}

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -495,6 +495,31 @@ public Builder audit(AuditConfiguration auditConfiguration) {
             return this;
         }
 
+        public Builder grpc(TestCertificates testCertificates) {
+            final String PRIVATE_KEY_GRPC_PASSWORD = "aWVV63OJ4qzZyPrBwl2MFny4ZV8lQRZchjL";
+            nodeOverrideSettingsBuilder.put("aux.transport.types", "experimental-secure-transport-grpc");
+            nodeOverrideSettingsBuilder.put("aux.transport.experimental-secure-transport-grpc.port", "9400-9500");
+            nodeOverrideSettingsBuilder.put("plugins.security.ssl.aux.experimental-secure-transport-grpc.enabled", true);
+            nodeOverrideSettingsBuilder.put(
+                "plugins.security.ssl.aux.experimental-secure-transport-grpc.pemcert_filepath",
+                testCertificates.getNodeCertificate(0).getAbsolutePath()
+            );
+            nodeOverrideSettingsBuilder.put(
+                "plugins.security.ssl.aux.experimental-secure-transport-grpc.pemkey_filepath",
+                testCertificates.getNodeKey(0, null).getAbsolutePath()
+            );
+            // nodeOverrideSettingsBuilder.put(
+            // "plugins.security.ssl.aux.experimental-secure-transport-grpc.pemkey_password",
+            // PRIVATE_KEY_GRPC_PASSWORD
+            // );
+            nodeOverrideSettingsBuilder.put("plugins.security.ssl.aux.experimental-secure-transport-grpc.clientauth_mode", "OPTIONAL");
+            nodeOverrideSettingsBuilder.put(
+                "plugins.security.ssl.aux.experimental-secure-transport-grpc.pemtrustedcas_filepath",
+                testCertificates.getRootCertificate().getAbsolutePath()
+            );
+            return this;
+        }
+
         public Builder internalAudit(AuditConfiguration auditConfiguration) {
             if (auditConfiguration != null) {
                 testSecurityConfig.audit(auditConfiguration);

src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java

@@ -232,6 +232,10 @@ default TestRestClient getRestClient(List<Header> headers, CertificateData useCe
         return createGenericClientRestClient(headers, useCertificateData, null);
     }
 
+    default TestGrpcClient getGrpcClient(TestCertificates testCertificates) {
+        return createGenericClientGrpcClient(testCertificates);
+    }
+
     default TestRestClient getSecurityDisabledRestClient() {
         return new TestRestClient(getHttpAddress(), List.of(), getSSLContext(null), null, false, false);
     }
@@ -244,6 +248,10 @@ default TestRestClient createGenericClientRestClient(
         return new TestRestClient(getHttpAddress(), headers, getSSLContext(useCertificateData), sourceInetAddress, true, false);
     }
 
+    default TestGrpcClient createGenericClientGrpcClient(TestCertificates testCertificates) {
+        return new TestGrpcClient(getHttpAddress(), "", getSSLContext(), testCertificates);
+    }
+
     default TestRestClient createGenericClientRestClient(TestRestClientConfiguration configuration) {
         return new TestRestClient(
             getHttpAddress(),