[GRPC] Make transport-grpc extensible to other plugins

CHANGELOG.md

@@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support system generated ingest pipelines for bulk update operations ([#18277](https://github.com/opensearch-project/OpenSearch/pull/18277)))
 - Added FS Health Check Failure metric ([#18435](https://github.com/opensearch-project/OpenSearch/pull/18435))
 - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509))
+- Make GRPC transport extensible to allow plugins to register and expose their own GRPC services [PR #18516](https://github.com/opensearch-project/OpenSearch/pull/18516)
 
 ### Changed
 - Create generic DocRequest to better categorize ActionRequests ([#18269](https://github.com/opensearch-project/OpenSearch/pull/18269)))

plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/GrpcPlugin.java

@@ -18,9 +18,13 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
+import org.opensearch.plugin.transport.grpc.proto.request.search.query.AbstractQueryBuilderProtoUtils;
+import org.opensearch.plugin.transport.grpc.proto.request.search.query.QueryBuilderProtoConverter;
+import org.opensearch.plugin.transport.grpc.proto.request.search.query.QueryBuilderProtoConverterRegistry;
 import org.opensearch.plugin.transport.grpc.services.DocumentServiceImpl;
 import org.opensearch.plugin.transport.grpc.services.SearchServiceImpl;
 import org.opensearch.plugin.transport.grpc.ssl.SecureNetty4GrpcServerTransport;
+import org.opensearch.plugins.ExtensiblePlugin;
 import org.opensearch.plugins.NetworkPlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.SecureAuxTransportSettingsProvider;
@@ -32,6 +36,7 @@
 import org.opensearch.transport.client.Client;
 import org.opensearch.watcher.ResourceWatcherService;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -55,15 +60,41 @@
 /**
  * Main class for the gRPC plugin.
  */
-public final class GrpcPlugin extends Plugin implements NetworkPlugin {
+public final class GrpcPlugin extends Plugin implements NetworkPlugin, ExtensiblePlugin {
 
     private Client client;
+    private final List<QueryBuilderProtoConverter> queryConverters = new ArrayList<>();
+    private QueryBuilderProtoConverterRegistry queryRegistry;
 
     /**
      * Creates a new GrpcPlugin instance.
      */
     public GrpcPlugin() {}
 
+    /**
+     * Loads extensions from other plugins.
+     * This method is called by the OpenSearch plugin system to load extensions from other plugins.
+     *
+     * @param loader The extension loader to use for loading extensions
+     */
+    @Override
+    public void loadExtensions(ExtensiblePlugin.ExtensionLoader loader) {
+        // Load query converters from other plugins
+        List<QueryBuilderProtoConverter> extensions = loader.loadExtensions(QueryBuilderProtoConverter.class);
+        if (extensions != null) {
+            queryConverters.addAll(extensions);
+        }
+    }
+
+    /**
+     * Get the list of query converters, including those loaded from extensions.
+     *
+     * @return The list of query converters
+     */
+    public List<QueryBuilderProtoConverter> getQueryConverters() {
+        return Collections.unmodifiableList(queryConverters);
+    }
+
     /**
      * Provides auxiliary transports for the plugin.
      * Creates and returns a map of transport names to transport suppliers.
@@ -88,6 +119,20 @@
         if (client == null) {
             throw new RuntimeException("client cannot be null");
         }
+
+        // Create the registry if it doesn't exist
+        if (queryRegistry == null) {
+            queryRegistry = new QueryBuilderProtoConverterRegistry();
+
+            // Register external converters
+            for (QueryBuilderProtoConverter converter : queryConverters) {
+                queryRegistry.registerConverter(converter);
+            }
+
+            // Set the registry in AbstractQueryBuilderProtoUtils
+            AbstractQueryBuilderProtoUtils.setRegistry(queryRegistry);
+        }
+
         List<BindableService> grpcServices = registerGRPCServices(new DocumentServiceImpl(client), new SearchServiceImpl(client));
         AuxTransport transport = new Netty4GrpcServerTransport(settings, grpcServices, networkService);
         return Collections.singletonMap(transport.settingKey(), () -> transport);
@@ -120,6 +165,20 @@
         if (client == null) {
             throw new RuntimeException("client cannot be null");
         }
+
+        // Create the registry if it doesn't exist
+        if (queryRegistry == null) {
+            queryRegistry = new QueryBuilderProtoConverterRegistry();
+
+            // Register external converters
+            for (QueryBuilderProtoConverter converter : queryConverters) {
+                queryRegistry.registerConverter(converter);
+            }
+
+            // Set the registry in AbstractQueryBuilderProtoUtils
+            AbstractQueryBuilderProtoUtils.setRegistry(queryRegistry);
+        }
+
         List<BindableService> grpcServices = registerGRPCServices(new DocumentServiceImpl(client), new SearchServiceImpl(client));
         AuxTransport transport = new SecureNetty4GrpcServerTransport(
             settings,
@@ -195,6 +254,17 @@
     ) {
         this.client = client;
 
+        // Create the registry
+        this.queryRegistry = new QueryBuilderProtoConverterRegistry();
+
+        // Register external converters
+        for (QueryBuilderProtoConverter converter : queryConverters) {
+            queryRegistry.registerConverter(converter);
+        }
+
+        // Set the registry in AbstractQueryBuilderProtoUtils
+        AbstractQueryBuilderProtoUtils.setRegistry(queryRegistry);
+
         return super.createComponents(
             client,
             clusterService,

plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/AbstractQueryBuilderProtoUtils.java

@@ -16,13 +16,44 @@
  * Utility class for converting Protocol Buffer query representations to OpenSearch QueryBuilder objects.
  * This class provides methods to parse different types of query containers and transform them
  * into their corresponding OpenSearch QueryBuilder implementations for search operations.
+ * <p>
+ * Supported query types include:
+ * <ul>
+ *   <li>MatchAll - Returns all documents</li>
+ *   <li>MatchNone - Returns no documents</li>
+ *   <li>Term - Exact term matching</li>
+ *   <li>Terms - Multiple terms matching</li>
+ *   <li>KNN - K-nearest neighbors vector search</li>
+ * </ul>
  */
 public class AbstractQueryBuilderProtoUtils {
 
+    private static QueryBuilderProtoConverterRegistry registry;
+
     private AbstractQueryBuilderProtoUtils() {
         // Utility class, no instances
     }
 
+    /**
+     * Sets the registry to use for query conversion.
+     * This method should be called during plugin initialization.
+     *
+     * @param registry The registry to use
+     */
+    public static void setRegistry(QueryBuilderProtoConverterRegistry registry) {
+        AbstractQueryBuilderProtoUtils.registry = registry;
+    }
+
+    /**
+     * Gets the current registry.
+     * This method is primarily for testing purposes.
+     *
+     * @return The current registry
+     */
+    public static QueryBuilderProtoConverterRegistry getRegistry() {
+        return registry;
+    }
+
     /**
      * Parse a query from its Protocol Buffer representation.
      * Similar to {@link AbstractQueryBuilder#parseInnerQueryBuilder(XContentParser)}, this method
@@ -34,22 +65,16 @@ private AbstractQueryBuilderProtoUtils() {
      * @throws UnsupportedOperationException if the query type is not supported
      */
     public static QueryBuilder parseInnerQueryBuilderProto(QueryContainer queryContainer) throws UnsupportedOperationException {
-        QueryBuilder result;
-
-        if (queryContainer.hasMatchAll()) {
-            result = MatchAllQueryBuilderProtoUtils.fromProto(queryContainer.getMatchAll());
-        } else if (queryContainer.hasMatchNone()) {
-            result = MatchNoneQueryBuilderProtoUtils.fromProto(queryContainer.getMatchNone());
-        } else if (queryContainer.getTermCount() > 0) {
-            result = TermQueryBuilderProtoUtils.fromProto(queryContainer.getTermMap());
-        } else if (queryContainer.hasTerms()) {
-            result = TermsQueryBuilderProtoUtils.fromProto(queryContainer.getTerms());
+        // Validate input
+        if (queryContainer == null) {
+            throw new IllegalArgumentException("Query container cannot be null");
         }
-        // TODO add more query types
-        else {
-            throw new UnsupportedOperationException("Search query type not supported yet.");
+
+        // If the registry is set, use it to convert the query
+        if (registry != null) {
+            return registry.fromProto(queryContainer);
         }
 
-        return result;
+        throw new IllegalArgumentException("Registry must be set.");
     }
 }

plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/MatchAllQueryBuilderProtoConverter.java

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.transport.grpc.proto.request.search.query;
+
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.QueryContainer;
+
+/**
+ * Converter for MatchAll queries.
+ * This class implements the QueryBuilderProtoConverter interface to provide MatchAll query support
+ * for the gRPC transport plugin.
+ */
+public class MatchAllQueryBuilderProtoConverter implements QueryBuilderProtoConverter {
+
+    /**
+     * Constructs a new MatchAllQueryBuilderProtoConverter.
+     */
+    public MatchAllQueryBuilderProtoConverter() {
+        // Default constructor
+    }
+
+    @Override
+    public boolean canHandle(QueryContainer queryContainer) {
+        return queryContainer != null && queryContainer.hasMatchAll();
+    }
+
+    @Override
+    public QueryBuilder fromProto(QueryContainer queryContainer) {
+        if (!canHandle(queryContainer)) {
+            throw new IllegalArgumentException("QueryContainer does not contain a MatchAll query");
+        }
+
+        return MatchAllQueryBuilderProtoUtils.fromProto(queryContainer.getMatchAll());
+    }
+}

plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/MatchNoneQueryBuilderProtoConverter.java

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.transport.grpc.proto.request.search.query;
+
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.QueryContainer;
+
+/**
+ * Converter for MatchNone queries.
+ * This class implements the QueryBuilderProtoConverter interface to provide MatchNone query support
+ * for the gRPC transport plugin.
+ */
+public class MatchNoneQueryBuilderProtoConverter implements QueryBuilderProtoConverter {
+
+    /**
+     * Constructs a new MatchNoneQueryBuilderProtoConverter.
+     */
+    public MatchNoneQueryBuilderProtoConverter() {
+        // Default constructor
+    }
+
+    @Override
+    public boolean canHandle(QueryContainer queryContainer) {
+        return queryContainer != null && queryContainer.hasMatchNone();
+    }
+
+    @Override
+    public QueryBuilder fromProto(QueryContainer queryContainer) {
+        if (!canHandle(queryContainer)) {
+            throw new IllegalArgumentException("QueryContainer does not contain a MatchNone query");
+        }
+
+        return MatchNoneQueryBuilderProtoUtils.fromProto(queryContainer.getMatchNone());
+    }
+}

plugins/transport-grpc/src/main/java/org/opensearch/plugin/transport/grpc/proto/request/search/query/QueryBuilderProtoConverter.java

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.plugin.transport.grpc.proto.request.search.query;
+
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.QueryContainer;
+
+/**
+ * Interface for converting protobuf query messages to OpenSearch QueryBuilder objects.
+ * External plugins can implement this interface to provide their own query types.
+ */
+public interface QueryBuilderProtoConverter {
+
+    /**
+     * Checks if this converter can handle the given query container.
+     *
+     * @param queryContainer The query container from the protobuf message
+     * @return true if this converter can handle the query container, false otherwise
+     */
+    boolean canHandle(QueryContainer queryContainer);
+
+    /**
+     * Converts a protobuf query container to an OpenSearch QueryBuilder.
+     *
+     * @param queryContainer The protobuf query container
+     * @return The corresponding OpenSearch QueryBuilder
+     * @throws IllegalArgumentException if the query cannot be converted
+     */
+    QueryBuilder fromProto(QueryContainer queryContainer);
+}