[GRPC] Add handling for the grpc.detailed_errors.enabled setting and global gRPC error_trace parameter

CHANGELOG.md

@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Add support for forward translog reading ([#20163](https://github.com/opensearch-project/OpenSearch/pull/20163))
 - Added public getter method in `SourceFieldMapper` to return excluded field ([#20205](https://github.com/opensearch-project/OpenSearch/pull/20205))
+- Add handling for the grpc.detailed_errors.enabled setting and global gRPC error_trace parameter ([#19644](https://github.com/opensearch-project/OpenSearch/pull/19644))
 - Add integ test for simulating node join left event when data node cluster state publication lag because the cluster applier thread being busy ([#19907](https://github.com/opensearch-project/OpenSearch/pull/19907)).
 - Relax jar hell check when extended plugins share transitive dependencies ([#20103](https://github.com/opensearch-project/OpenSearch/pull/20103))
 - Added public getter method in `SourceFieldMapper` to return included field ([#20290](https://github.com/opensearch-project/OpenSearch/pull/20290))

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/GrpcPlugin.java

@@ -42,6 +42,7 @@
 import org.opensearch.transport.grpc.spi.GrpcServiceFactory;
 import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverter;
 import org.opensearch.transport.grpc.ssl.SecureNetty4GrpcServerTransport;
+import org.opensearch.transport.grpc.util.GrpcParamsHandler;
 import org.opensearch.watcher.ResourceWatcherService;
 
 import java.util.ArrayList;
@@ -58,6 +59,7 @@
 
 import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.GRPC_TRANSPORT_SETTING_KEY;
 import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.SETTING_GRPC_BIND_HOST;
+import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.SETTING_GRPC_DETAILED_ERRORS_ENABLED;
 import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.SETTING_GRPC_EXECUTOR_COUNT;
 import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.SETTING_GRPC_HOST;
 import static org.opensearch.transport.grpc.Netty4GrpcServerTransport.SETTING_GRPC_KEEPALIVE_TIMEOUT;
@@ -78,7 +80,9 @@
 public final class GrpcPlugin extends Plugin implements NetworkPlugin, ExtensiblePlugin {
     private static final Logger logger = LogManager.getLogger(GrpcPlugin.class);
 
-    /** The name of the gRPC thread pool */
+    /**
+     * The name of the gRPC thread pool
+     */
     public static final String GRPC_THREAD_POOL_NAME = "grpc";
 
     private final List<QueryBuilderProtoConverter> queryConverters = new ArrayList<>();
@@ -182,6 +186,7 @@ public Map<String, Supplier<AuxTransport>> getAuxTransports(
         }
 
         return Collections.singletonMap(GRPC_TRANSPORT_SETTING_KEY, () -> {
+            GrpcParamsHandler.initialize(settings);
             List<BindableService> grpcServices = new ArrayList<>(
                 List.of(new DocumentServiceImpl(client), new SearchServiceImpl(client, queryUtils))
             );
@@ -233,6 +238,7 @@ public Map<String, Supplier<AuxTransport>> getSecureAuxTransports(
             throw new RuntimeException("createComponents must be called first to initialize server provided resources.");
         }
         return Collections.singletonMap(GRPC_SECURE_TRANSPORT_SETTING_KEY, () -> {
+            GrpcParamsHandler.initialize(settings);
             List<BindableService> grpcServices = new ArrayList<>(
                 List.of(new DocumentServiceImpl(client), new SearchServiceImpl(client, queryUtils))
             );
@@ -282,7 +288,8 @@ public List<Setting<?>> getSettings() {
             SETTING_GRPC_MAX_MSG_SIZE,
             SETTING_GRPC_MAX_CONNECTION_AGE,
             SETTING_GRPC_MAX_CONNECTION_IDLE,
-            SETTING_GRPC_KEEPALIVE_TIMEOUT
+            SETTING_GRPC_KEEPALIVE_TIMEOUT,
+            SETTING_GRPC_DETAILED_ERRORS_ENABLED
         );
     }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/Netty4GrpcServerTransport.java

@@ -187,6 +187,18 @@ public class Netty4GrpcServerTransport extends AuxTransport {
         Setting.Property.NodeScope
     );
 
+    /**
+     * Enables detailed error message for gRPC requests. If enabled, returns detailed
+     * error information when requested via {@code error_trace} parameter in the request scope.
+     * Otherwise, only a summary is generated. The case when the detailed error trace is requested
+     * but server explicitly turned it off, the error response is generated.
+     */
+    public static final Setting<Boolean> SETTING_GRPC_DETAILED_ERRORS_ENABLED = Setting.boolSetting(
+        "grpc.detailed_errors.enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+
     /**
      * Port range on which servers bind.
      */

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/listeners/BulkRequestActionListener.java

@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.bulk.BulkResponse;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.transport.grpc.proto.response.document.bulk.BulkResponseProtoUtils;
 import org.opensearch.transport.grpc.util.GrpcErrorHandler;
 
@@ -26,15 +27,18 @@
 public class BulkRequestActionListener implements ActionListener<BulkResponse> {
     private static final Logger logger = LogManager.getLogger(BulkRequestActionListener.class);
     private final StreamObserver<org.opensearch.protobufs.BulkResponse> responseObserver;
+    private final GlobalParams params;
 
     /**
      * Creates a new BulkRequestActionListener.
      *
      * @param responseObserver The gRPC stream observer to send the response back to the client
+     * @param params parameters that are going to change how responses and errors are handled
      */
-    public BulkRequestActionListener(StreamObserver<org.opensearch.protobufs.BulkResponse> responseObserver) {
+    public BulkRequestActionListener(StreamObserver<org.opensearch.protobufs.BulkResponse> responseObserver, GlobalParams params) {
         super();
         this.responseObserver = responseObserver;
+        this.params = params;
     }
 
     /**
@@ -47,12 +51,12 @@ public BulkRequestActionListener(StreamObserver<org.opensearch.protobufs.BulkRes
     public void onResponse(org.opensearch.action.bulk.BulkResponse response) {
         // Bulk execution succeeded. Convert the opensearch internal response to protobuf
         try {
-            org.opensearch.protobufs.BulkResponse protoResponse = BulkResponseProtoUtils.toProto(response);
+            org.opensearch.protobufs.BulkResponse protoResponse = BulkResponseProtoUtils.toProto(response, params);
             responseObserver.onNext(protoResponse);
             responseObserver.onCompleted();
         } catch (RuntimeException | IOException e) {
             logger.error("Failed to convert bulk response to protobuf: " + e.getMessage());
-            StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e);
+            StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e, params);
             responseObserver.onError(grpcError);
         }
     }
@@ -66,7 +70,7 @@ public void onResponse(org.opensearch.action.bulk.BulkResponse response) {
     @Override
     public void onFailure(Exception e) {
         logger.error("BulkRequestActionListener failed to process bulk request: " + e.getMessage());
-        StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e);
+        StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e, params);
         responseObserver.onError(grpcError);
     }
 }

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/listeners/SearchRequestActionListener.java

@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.transport.grpc.proto.response.search.SearchResponseProtoUtils;
 import org.opensearch.transport.grpc.util.GrpcErrorHandler;
 
@@ -27,35 +28,38 @@ public class SearchRequestActionListener implements ActionListener<SearchRespons
     private static final Logger logger = LogManager.getLogger(SearchRequestActionListener.class);
 
     private final StreamObserver<org.opensearch.protobufs.SearchResponse> responseObserver;
+    private final GlobalParams params;
 
     /**
      * Constructs a new SearchRequestActionListener.
      *
      * @param responseObserver the gRPC stream observer to send the search response to
+     * @param params parameters that are going to change how responses and errors are handled
      */
-    public SearchRequestActionListener(StreamObserver<org.opensearch.protobufs.SearchResponse> responseObserver) {
+    public SearchRequestActionListener(StreamObserver<org.opensearch.protobufs.SearchResponse> responseObserver, GlobalParams params) {
         super();
         this.responseObserver = responseObserver;
+        this.params = params;
     }
 
     @Override
     public void onResponse(SearchResponse response) {
         // Search execution succeeded. Convert the opensearch internal response to protobuf
         try {
-            org.opensearch.protobufs.SearchResponse protoResponse = SearchResponseProtoUtils.toProto(response);
+            org.opensearch.protobufs.SearchResponse protoResponse = SearchResponseProtoUtils.toProto(response, params);
             responseObserver.onNext(protoResponse);
             responseObserver.onCompleted();
         } catch (RuntimeException | IOException e) {
             logger.error("Failed to convert search response to protobuf: " + e.getMessage());
-            StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e);
+            StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e, params);
             responseObserver.onError(grpcError);
         }
     }
 
     @Override
     public void onFailure(Exception e) {
         logger.debug("SearchRequestActionListener failed to process search request: " + e.getMessage());
-        StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e);
+        StatusRuntimeException grpcError = GrpcErrorHandler.convertToGrpcError(e, params);
         responseObserver.onError(grpcError);
     }
 }

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/document/bulk/BulkRequestProtoUtils.java

@@ -10,6 +10,7 @@
 
 import org.opensearch.action.bulk.BulkShardRequest;
 import org.opensearch.protobufs.BulkRequest;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.document.RestBulkAction;
 import org.opensearch.search.fetch.subphase.FetchSourceContext;
@@ -90,9 +91,10 @@ public static org.opensearch.action.bulk.BulkRequest prepareRequest(BulkRequest
             throw new UnsupportedOperationException("type param is not supported");
         }
 
-        // TODO support global_params
-        if (request.hasGlobalParams()) {
-            throw new UnsupportedOperationException("global_params param is not supported yet");
+        // TODO support global_params parameters
+        GlobalParams params = request.getGlobalParams();
+        if (params.hasHuman() || params.getFilterPathCount() > 0) {
+            throw new UnsupportedOperationException("global_params.human or filter_path params are not supported yet");
         }
 
         return bulkRequest;

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/SearchRequestProtoUtils.java

@@ -15,6 +15,7 @@
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.protobufs.SearchRequest;
 import org.opensearch.protobufs.SearchRequestBody;
 import org.opensearch.rest.RestRequest;
@@ -189,9 +190,10 @@ protected static void parseSearchRequest(
             throw new UnsupportedOperationException("typed_keys param is not supported yet");
         }
 
-        // TODO support global_params
-        if (request.hasGlobalParams()) {
-            throw new UnsupportedOperationException("global_params param is not supported yet");
+        // TODO support global_params parameters
+        GlobalParams params = request.getGlobalParams();
+        if (params.hasHuman() || params.getFilterPathCount() > 0) {
+            throw new UnsupportedOperationException("global_params.human or filter_path params are not supported yet");
         }
     }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/bulk/BulkItemResponseProtoUtils.java

@@ -14,6 +14,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.get.GetResult;
 import org.opensearch.protobufs.ErrorCause;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.protobufs.ResponseItem;
 import org.opensearch.transport.grpc.proto.response.document.common.DocWriteResponseProtoUtils;
 import org.opensearch.transport.grpc.proto.response.document.get.GetResultProtoUtils;
@@ -39,16 +40,17 @@ private BulkItemResponseProtoUtils() {
      *
      *
      * @param response The BulkItemResponse to convert
+     * @param params The global gRPC request parameters
      * @return A Protocol Buffer ResponseItem representation
      * @throws IOException if there's an error during conversion
      *
      */
-    public static ResponseItem toProto(BulkItemResponse response) throws IOException {
+    public static ResponseItem toProto(BulkItemResponse response, GlobalParams params) throws IOException {
         ResponseItem.Builder responseItemBuilder;
 
         if (response.isFailed() == false) {
             DocWriteResponse docResponse = response.getResponse();
-            responseItemBuilder = DocWriteResponseProtoUtils.toProto(docResponse);
+            responseItemBuilder = DocWriteResponseProtoUtils.toProto(docResponse, params);
 
             int grpcStatusCode = RestToGrpcStatusConverter.getGrpcStatusCode(docResponse.status());
             responseItemBuilder.setStatus(grpcStatusCode);
@@ -63,7 +65,7 @@ public static ResponseItem toProto(BulkItemResponse response) throws IOException
             int grpcStatusCode = RestToGrpcStatusConverter.getGrpcStatusCode(failure.getStatus());
             responseItemBuilder.setStatus(grpcStatusCode);
 
-            ErrorCause errorCause = OpenSearchExceptionProtoUtils.generateThrowableProto(failure.getCause());
+            ErrorCause errorCause = OpenSearchExceptionProtoUtils.generateThrowableProto(failure.getCause(), params);
             responseItemBuilder.setError(errorCause);
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/bulk/BulkResponseProtoUtils.java

@@ -11,6 +11,7 @@
 import org.opensearch.action.bulk.BulkResponse;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.protobufs.GlobalParams;
 
 import java.io.IOException;
 
@@ -30,10 +31,11 @@ private BulkResponseProtoUtils() {
      * This method is equivalent to {@link BulkResponse#toXContent(XContentBuilder, ToXContent.Params)}
      *
      * @param response The BulkResponse to convert
+     * @param params The global gRPC request parameters
      * @return A Protocol Buffer BulkResponse representation
      * @throws IOException if there's an error during conversion
      */
-    public static org.opensearch.protobufs.BulkResponse toProto(BulkResponse response) throws IOException {
+    public static org.opensearch.protobufs.BulkResponse toProto(BulkResponse response, GlobalParams params) throws IOException {
         // System.out.println("=== grpc bulk response=" + response.toString());
 
         org.opensearch.protobufs.BulkResponse.Builder bulkResponse = org.opensearch.protobufs.BulkResponse.newBuilder();
@@ -51,7 +53,7 @@ public static org.opensearch.protobufs.BulkResponse toProto(BulkResponse respons
 
         // Add individual item responses for each operation in the bulk request
         for (BulkItemResponse bulkItemResponse : response.getItems()) {
-            org.opensearch.protobufs.ResponseItem responseItem = BulkItemResponseProtoUtils.toProto(bulkItemResponse);
+            org.opensearch.protobufs.ResponseItem responseItem = BulkItemResponseProtoUtils.toProto(bulkItemResponse, params);
             org.opensearch.protobufs.Item.Builder itemBuilder = org.opensearch.protobufs.Item.newBuilder();
 
             // Wrap ResponseItem in Item based on operation type

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/common/DocWriteResponseProtoUtils.java

@@ -10,6 +10,7 @@
 import org.opensearch.action.DocWriteResponse;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.protobufs.ResponseItem;
 import org.opensearch.protobufs.ShardInfo;
 
@@ -31,10 +32,11 @@ private DocWriteResponseProtoUtils() {
      * This method is equivalent to the {@link DocWriteResponse#innerToXContent(XContentBuilder, ToXContent.Params)}
      *
      * @param response The DocWriteResponse to convert
+     * @param params The global gRPC request parameters
      * @return A ResponseItem.Builder with the DocWriteResponse data
      *
      */
-    public static ResponseItem.Builder toProto(DocWriteResponse response) throws IOException {
+    public static ResponseItem.Builder toProto(DocWriteResponse response, GlobalParams params) throws IOException {
         ResponseItem.Builder responseItem = ResponseItem.newBuilder();
 
         // Set the index name
@@ -56,7 +58,7 @@ public static ResponseItem.Builder toProto(DocWriteResponse response) throws IOE
             responseItem.setForcedRefresh(true);
         }
         // Handle shard information
-        ShardInfo shardInfo = ShardInfoProtoUtils.toProto(response.getShardInfo());
+        ShardInfo shardInfo = ShardInfoProtoUtils.toProto(response.getShardInfo(), params);
         responseItem.setXShards(shardInfo);
 
         // Set sequence number and primary term if available

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/common/ShardInfoProtoUtils.java

@@ -10,6 +10,7 @@
 import org.opensearch.action.support.replication.ReplicationResponse;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.protobufs.GlobalParams;
 import org.opensearch.protobufs.ShardFailure;
 import org.opensearch.protobufs.ShardInfo;
 import org.opensearch.transport.grpc.proto.response.exceptions.opensearchexception.OpenSearchExceptionProtoUtils;
@@ -30,18 +31,19 @@ private ShardInfoProtoUtils() {
      * Similar to {@link ReplicationResponse.ShardInfo#toXContent(XContentBuilder, ToXContent.Params)}
      *
      * @param shardInfo The shard information to convert to protobuf format
+     * @param params The global gRPC request parameters
      * @return The protobuf representation of the shard information
      * @throws IOException If there's an error during conversion
      */
-    public static ShardInfo toProto(ReplicationResponse.ShardInfo shardInfo) throws IOException {
+    public static ShardInfo toProto(ReplicationResponse.ShardInfo shardInfo, GlobalParams params) throws IOException {
         ShardInfo.Builder shardInfoBuilder = ShardInfo.newBuilder();
         shardInfoBuilder.setTotal(shardInfo.getTotal());
         shardInfoBuilder.setSuccessful(shardInfo.getSuccessful());
         shardInfoBuilder.setFailed(shardInfo.getFailed());
 
         // Add any shard failures
         for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
-            shardInfoBuilder.addFailures(toProto(failure));
+            shardInfoBuilder.addFailures(toProto(failure, params));
         }
 
         return shardInfoBuilder.build();
@@ -55,12 +57,12 @@ public static ShardInfo toProto(ReplicationResponse.ShardInfo shardInfo) throws
      * @return The protobuf representation of the shard failure
      * @throws IOException If there's an error during conversion
      */
-    private static ShardFailure toProto(ReplicationResponse.ShardInfo.Failure failure) throws IOException {
+    private static ShardFailure toProto(ReplicationResponse.ShardInfo.Failure failure, GlobalParams params) throws IOException {
         ShardFailure.Builder shardFailure = ShardFailure.newBuilder();
         shardFailure.setIndex(failure.index());
         shardFailure.setShard(failure.shardId());
         shardFailure.setNode(failure.nodeId());
-        shardFailure.setReason(OpenSearchExceptionProtoUtils.generateThrowableProto(failure.getCause()));
+        shardFailure.setReason(OpenSearchExceptionProtoUtils.generateThrowableProto(failure.getCause(), params));
         shardFailure.setStatus(failure.status().name());
         shardFailure.setPrimary(failure.primary());
         return shardFailure.build();