[GRPC] Upgrade to protobufs 0.24.0 and fix compilation errors

CHANGELOG.md

@@ -64,6 +64,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the IndexPressutreStats, DeviceStats and TransportStats class to use the Builder pattern instead of constructors ([#19991](https://github.com/opensearch-project/OpenSearch/pull/19991))
 - Refactor the Cache.CacheStats class to use the Builder pattern instead of constructors ([#20015](https://github.com/opensearch-project/OpenSearch/pull/20015))
 - Refactor the HttpStats, ScriptStats, AdaptiveSelectionStats and OsStats class to use the Builder pattern instead of constructors ([#20014](https://github.com/opensearch-project/OpenSearch/pull/20014))
+- Bump opensearch-protobufs dependency to 0.24.0 and update transport-grpc module compatibility ([#20059](https://github.com/opensearch-project/OpenSearch/pull/20059))
+
 
 ### Fixed
 - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012))

gradle/libs.versions.toml

@@ -22,7 +22,7 @@ kotlin            = "1.7.10"
 antlr4            = "4.13.1"
 guava             = "33.2.1-jre"
 gson              = "2.13.2"
-opensearchprotobufs = "0.23.0"
+opensearchprotobufs = "0.24.0"
 protobuf          = "3.25.8"
 jakarta_annotation = "1.3.5"
 google_http_client = "1.44.1"

modules/transport-grpc/licenses/protobufs-0.24.0.jar.sha1

@@ -0,0 +1 @@
+4330dd089f853e345aa137708490aec841ffadc0

modules/transport-grpc/spi/licenses/protobufs-0.24.0.jar.sha1

@@ -0,0 +1 @@
+4330dd089f853e345aa137708490aec841ffadc0

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/common/FetchSourceContextProtoUtils.java

@@ -45,11 +45,11 @@ public static FetchSourceContext parseFromProtoRequest(org.opensearch.protobufs.
         // Set up source context if source parameters are provided
         if (request.hasXSource()) {
             switch (request.getXSource().getSourceConfigParamCase()) {
-                case BOOL:
-                    fetchSource = request.getXSource().getBool();
+                case FETCH:
+                    fetchSource = request.getXSource().getFetch();
                     break;
-                case STRING_ARRAY:
-                    sourceIncludes = request.getXSource().getStringArray().getStringArrayList().toArray(new String[0]);
+                case FIELDS:
+                    sourceIncludes = request.getXSource().getFields().getStringArrayList().toArray(new String[0]);
                     break;
                 default:
                     throw new UnsupportedOperationException("Invalid sourceConfig provided.");
@@ -84,10 +84,10 @@ public static FetchSourceContext parseFromProtoRequest(org.opensearch.protobufs.
         if (request.hasXSource()) {
             SourceConfigParam source = request.getXSource();
 
-            if (source.hasBool()) {
-                fetchSource = source.getBool();
+            if (source.hasFetch()) {
+                fetchSource = source.getFetch();
             } else {
-                sourceIncludes = source.getStringArray().getStringArrayList().toArray(new String[0]);
+                sourceIncludes = source.getFields().getStringArrayList().toArray(new String[0]);
             }
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/common/ScriptProtoUtils.java

@@ -76,7 +76,7 @@ public static Script parseInlineScript(InlineScript inlineScript, String default
 
         ScriptType type = ScriptType.INLINE;
 
-        String lang = parseScriptLanguage(inlineScript.getLang(), defaultLang);
+        String lang = inlineScript.hasLang() ? parseScriptLanguage(inlineScript.getLang(), defaultLang) : defaultLang;
         String idOrCode = inlineScript.getSource();
 
         Map<String, String> options = inlineScript.getOptionsMap();

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/document/bulk/ActiveShardCountProtoUtils.java

@@ -38,17 +38,17 @@ private ActiveShardCountProtoUtils() {
     public static ActiveShardCount parseProto(WaitForActiveShards waitForActiveShards) {
 
         switch (waitForActiveShards.getWaitForActiveShardsCase()) {
-            case WAIT_FOR_ACTIVE_SHARD_OPTIONS:
-                switch (waitForActiveShards.getWaitForActiveShardOptions()) {
+            case OPTION:
+                switch (waitForActiveShards.getOption()) {
                     case WAIT_FOR_ACTIVE_SHARD_OPTIONS_ALL:
                         return ActiveShardCount.ALL;
                     case WAIT_FOR_ACTIVE_SHARD_OPTIONS_NULL:
                     case WAIT_FOR_ACTIVE_SHARD_OPTIONS_UNSPECIFIED:
                     default:
                         return ActiveShardCount.DEFAULT;
                 }
-            case INT32:
-                return ActiveShardCount.from(waitForActiveShards.getInt32());
+            case COUNT:
+                return ActiveShardCount.from(waitForActiveShards.getCount());
             case WAITFORACTIVESHARDS_NOT_SET:
             default:
                 return ActiveShardCount.DEFAULT;

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/DerivedFieldProtoUtils.java

@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.transport.grpc.proto.request.search;
+
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.transport.grpc.proto.request.common.ObjectMapProtoUtils;
+import org.opensearch.transport.grpc.proto.request.common.ScriptProtoUtils;
+
+/**
+ * Utility class for converting DerivedField Protocol Buffers to OpenSearch DerivedField objects.
+ * This class handles the conversion of Protocol Buffer representations to their
+ * corresponding OpenSearch derived field objects, matching the REST side pattern.
+ *
+ * @see org.opensearch.index.mapper.DerivedFieldMapper.Builder#build(org.opensearch.index.mapper.Mapper.BuilderContext)
+ * @see org.opensearch.index.mapper.DefaultDerivedFieldResolver#initDerivedFieldTypes(java.util.Map, java.util.List)
+ */
+public class DerivedFieldProtoUtils {
+
+    private DerivedFieldProtoUtils() {
+        // Utility class, no instances
+    }
+
+    /**
+     * Converts a Protocol Buffer DerivedField to its corresponding OpenSearch DerivedField.
+     * Uses the same pattern as REST side: always use simple constructor, then conditionally
+     * set optional fields based on presence (matches DerivedFieldMapper.Builder.build() pattern).
+     *
+     *
+     * <p>This matches the REST side's {@code Parameter.isConfigured()} pattern in
+     * {@link org.opensearch.index.mapper.DerivedFieldMapper.Builder#build}, where optional
+     * fields are only set if they were present in the request.</p>
+     *
+     * @param name The name of the derived field
+     * @param derivedFieldProto The Protocol Buffer DerivedField to convert
+     * @return The corresponding OpenSearch DerivedField with optional fields set
+     * @see org.opensearch.index.mapper.DerivedFieldMapper.Builder#build(org.opensearch.index.mapper.Mapper.BuilderContext)
+     * @see org.opensearch.index.mapper.ParametrizedFieldMapper.Parameter#isConfigured()
+     */
+    public static DerivedField fromProto(String name, org.opensearch.protobufs.DerivedField derivedFieldProto) {
+        // Always use simple constructor first (matches REST side pattern in DerivedFieldMapper.Builder.build())
+        DerivedField derivedField = new DerivedField(
+            name,
+            derivedFieldProto.getType(),
+            ScriptProtoUtils.parseFromProtoRequest(derivedFieldProto.getScript())
+        );
+
+        // Conditionally set optional fields based on presence (matches REST side's isConfigured() pattern)
+        if (derivedFieldProto.hasProperties()) {
+            derivedField.setProperties(ObjectMapProtoUtils.fromProto(derivedFieldProto.getProperties()));
+        }
+        if (derivedFieldProto.hasPrefilterField()) {
+            derivedField.setPrefilterField(derivedFieldProto.getPrefilterField());
+        }
+        if (derivedFieldProto.hasFormat()) {
+            derivedField.setFormat(derivedFieldProto.getFormat());
+        }
+        if (derivedFieldProto.hasIgnoreMalformed()) {
+            derivedField.setIgnoreMalformed(derivedFieldProto.getIgnoreMalformed());
+        }
+
+        return derivedField;
+    }
+}

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/HighlightBuilderProtoUtils.java

@@ -128,8 +128,8 @@ static HighlightBuilder fromProto(Highlight highlightProto, QueryBuilderProtoCon
             highlightBuilder.phraseLimit(highlightProto.getPhraseLimit());
         }
 
-        if (highlightProto.hasMaxAnalyzedOffset()) {
-            highlightBuilder.maxAnalyzerOffset(highlightProto.getMaxAnalyzedOffset());
+        if (highlightProto.hasMaxAnalyzerOffset()) {
+            highlightBuilder.maxAnalyzerOffset(highlightProto.getMaxAnalyzerOffset());
         }
 
         if (highlightProto.hasOptions()) {
@@ -244,8 +244,12 @@ static HighlightBuilder fromProto(Highlight highlightProto, QueryBuilderProtoCon
                 if (fieldProto.hasOptions()) {
                     fieldBuilder.options(ObjectMapProtoUtils.fromProto(fieldProto.getOptions()));
                 }
-                if (fieldProto.hasMaxAnalyzedOffset()) {
-                    fieldBuilder.maxAnalyzerOffset(fieldProto.getMaxAnalyzedOffset());
+                if (fieldProto.hasMaxAnalyzerOffset()) {
+                    fieldBuilder.maxAnalyzerOffset(fieldProto.getMaxAnalyzerOffset());
+                }
+
+                if (fieldProto.hasForceSource()) {
+                    fieldBuilder.forceSource(fieldProto.getForceSource());
                 }
 
                 highlightBuilder.field(fieldBuilder);

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/SearchSourceBuilderProtoUtils.java

@@ -186,12 +186,31 @@ private static void parseNonQueryFields(
         if (protoRequest.getDerivedCount() > 0) {
             for (Map.Entry<String, DerivedField> entry : protoRequest.getDerivedMap().entrySet()) {
                 String name = entry.getKey();
-                DerivedField derivedField = entry.getValue();
-                searchSourceBuilder.derivedField(
-                    name,
-                    derivedField.getType(),
-                    ScriptProtoUtils.parseFromProtoRequest(derivedField.getScript())
-                );
+                DerivedField derivedFieldProto = entry.getValue();
+
+                // Convert protobuf DerivedField to OpenSearch DerivedField using the REST side pattern
+                // This uses simple constructor + conditional setters (matches DerivedFieldMapper.Builder.build())
+                org.opensearch.index.mapper.DerivedField derivedField = DerivedFieldProtoUtils.fromProto(name, derivedFieldProto);
+
+                // Add to SearchSourceBuilder - check if any optional fields are set to choose the right method
+                if (derivedField.getProperties() != null
+                    || derivedField.getPrefilterField() != null
+                    || derivedField.getFormat() != null
+                    || derivedField.getIgnoreMalformed()) {
+                    // Use full constructor when optional fields are present
+                    searchSourceBuilder.derivedField(
+                        derivedField.getName(),
+                        derivedField.getType(),
+                        derivedField.getScript(),
+                        derivedField.getProperties(),
+                        derivedField.getPrefilterField(),
+                        derivedField.getFormat(),
+                        derivedField.getIgnoreMalformed() ? Boolean.TRUE : null
+                    );
+                } else {
+                    // Use simple constructor when no optional fields
+                    searchSourceBuilder.derivedField(derivedField.getName(), derivedField.getType(), derivedField.getScript());
+                }
             }
         }
         if (protoRequest.getDocvalueFieldsCount() > 0) {

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/MatchBoolPrefixQueryBuilderProtoUtils.java

@@ -57,7 +57,8 @@ static MatchBoolPrefixQueryBuilder fromProto(MatchBoolPrefixQuery matchBoolPrefi
             analyzer = matchBoolPrefixQueryProto.getAnalyzer();
         }
 
-        if (matchBoolPrefixQueryProto.getOperator() != org.opensearch.protobufs.Operator.OPERATOR_UNSPECIFIED) {
+        if (matchBoolPrefixQueryProto.hasOperator()
+            && matchBoolPrefixQueryProto.getOperator() != org.opensearch.protobufs.Operator.OPERATOR_UNSPECIFIED) {
             operator = OperatorProtoUtils.fromEnum(matchBoolPrefixQueryProto.getOperator());
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/MatchQueryBuilderProtoUtils.java

@@ -87,7 +87,7 @@ static MatchQueryBuilder fromProto(org.opensearch.protobufs.MatchQuery matchQuer
             maxExpansions = matchQueryProto.getMaxExpansions();
         }
 
-        if (matchQueryProto.getOperator() != org.opensearch.protobufs.Operator.OPERATOR_UNSPECIFIED) {
+        if (matchQueryProto.hasOperator() && matchQueryProto.getOperator() != org.opensearch.protobufs.Operator.OPERATOR_UNSPECIFIED) {
             operator = OperatorProtoUtils.fromEnum(matchQueryProto.getOperator());
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/MultiMatchQueryBuilderProtoUtils.java

@@ -14,6 +14,7 @@
 import org.opensearch.index.search.MatchQuery;
 import org.opensearch.protobufs.MultiMatchQuery;
 import org.opensearch.transport.grpc.proto.request.search.OperatorProtoUtils;
+import org.opensearch.transport.grpc.util.ProtobufEnumUtils;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -133,8 +134,10 @@ static MultiMatchQueryBuilder fromProto(MultiMatchQuery multiMatchQueryProto) {
             }
         }
 
-        if (multiMatchQueryProto.hasFuzzyRewrite()) {
-            fuzzyRewrite = multiMatchQueryProto.getFuzzyRewrite();
+        if (multiMatchQueryProto.hasFuzzyRewrite()
+            && multiMatchQueryProto
+                .getFuzzyRewrite() != org.opensearch.protobufs.MultiTermQueryRewrite.MULTI_TERM_QUERY_REWRITE_UNSPECIFIED) {
+            fuzzyRewrite = ProtobufEnumUtils.convertToString(multiMatchQueryProto.getFuzzyRewrite());
         }
 
         if (multiMatchQueryProto.hasTieBreaker()) {

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/sort/ScoreSortProtoUtils.java

@@ -43,7 +43,7 @@ public static ScoreSortBuilder fromProto(ScoreSort scoreSort) {
 
         ScoreSortBuilder builder = new ScoreSortBuilder();
 
-        if (scoreSort.getOrder() != org.opensearch.protobufs.SortOrder.SORT_ORDER_UNSPECIFIED) {
+        if (scoreSort.hasOrder() && scoreSort.getOrder() != org.opensearch.protobufs.SortOrder.SORT_ORDER_UNSPECIFIED) {
             SortOrder order = SortOrder.fromString(ProtobufEnumUtils.convertToString(scoreSort.getOrder()));
             builder.order(order);
         }

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/bulk/BulkItemResponseProtoUtils.java

@@ -14,8 +14,6 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.get.GetResult;
 import org.opensearch.protobufs.ErrorCause;
-import org.opensearch.protobufs.Id;
-import org.opensearch.protobufs.NullValue;
 import org.opensearch.protobufs.ResponseItem;
 import org.opensearch.transport.grpc.proto.response.document.common.DocWriteResponseProtoUtils;
 import org.opensearch.transport.grpc.proto.response.document.get.GetResultProtoUtils;
@@ -59,10 +57,8 @@ public static ResponseItem toProto(BulkItemResponse response) throws IOException
             responseItemBuilder = ResponseItem.newBuilder();
 
             responseItemBuilder.setXIndex(failure.getIndex());
-            if (response.getId().isEmpty()) {
-                responseItemBuilder.setXId(Id.newBuilder().setNullValue(NullValue.NULL_VALUE_NULL).build());
-            } else {
-                responseItemBuilder.setXId(Id.newBuilder().setString(response.getId()).build());
+            if (response.getId() != null && !response.getId().isEmpty()) {
+                responseItemBuilder.setXId(response.getId());
             }
             int grpcStatusCode = RestToGrpcStatusConverter.getGrpcStatusCode(failure.getStatus());
             responseItemBuilder.setStatus(grpcStatusCode);

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/common/DocWriteResponseProtoUtils.java

@@ -10,8 +10,6 @@
 import org.opensearch.action.DocWriteResponse;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.protobufs.Id;
-import org.opensearch.protobufs.NullValue;
 import org.opensearch.protobufs.ResponseItem;
 import org.opensearch.protobufs.ShardInfo;
 
@@ -43,10 +41,8 @@ public static ResponseItem.Builder toProto(DocWriteResponse response) throws IOE
         responseItem.setXIndex(response.getIndex());
 
         // Handle document ID (can be null in some cases)
-        if (response.getId().isEmpty()) {
-            responseItem.setXId(Id.newBuilder().setNullValue(NullValue.NULL_VALUE_NULL).build());
-        } else {
-            responseItem.setXId(Id.newBuilder().setString(response.getId()).build());
+        if (response.getId() != null && !response.getId().isEmpty()) {
+            responseItem.setXId(response.getId());
         }
 
         // Set document version

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/document/get/GetResultProtoUtils.java

@@ -14,7 +14,6 @@
 import org.opensearch.index.get.GetResult;
 import org.opensearch.index.mapper.IgnoredFieldMapper;
 import org.opensearch.index.seqno.SequenceNumbers;
-import org.opensearch.protobufs.Id;
 import org.opensearch.protobufs.InlineGetDictUserDefined;
 import org.opensearch.protobufs.ObjectMap;
 import org.opensearch.protobufs.ResponseItem;
@@ -43,9 +42,8 @@ public static ResponseItem.Builder toProto(GetResult getResult, ResponseItem.Bui
         // Reuse the builder passed in by reference
         responseItemBuilder.setXIndex(getResult.getIndex());
 
-        // Avoid creating a new Id builder for each call
-        Id id = Id.newBuilder().setString(getResult.getId()).build();
-        responseItemBuilder.setXId(id);
+        // Set document ID
+        responseItemBuilder.setXId(getResult.getId());
 
         // Create the inline get dict builder only once
         InlineGetDictUserDefined.Builder inlineGetDictUserDefinedBuilder = InlineGetDictUserDefined.newBuilder();
@@ -74,7 +72,7 @@ public static ResponseItem.Builder toProto(GetResult getResult, ResponseItem.Bui
     public static void toProtoEmbedded(GetResult getResult, InlineGetDictUserDefined.Builder builder) {
         // Set sequence number and primary term if available
         if (getResult.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
-            builder.setSeqNo(getResult.getSeqNo());
+            builder.setXSeqNo(getResult.getSeqNo());
             builder.setXPrimaryTerm(getResult.getPrimaryTerm());
         }
 

modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/response/exceptions/opensearchexception/OpenSearchExceptionProtoUtils.java

@@ -127,23 +127,31 @@ public static ErrorCause innerToProto(
             errorCauseBuilder.setReason(message);
         }
 
+        // Build metadata ObjectMap
+        ObjectMap.Builder metadataBuilder = ObjectMap.newBuilder();
+
         // Add custom metadata fields propogated by the child classes of OpenSearchException
         for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
             Map.Entry<String, ObjectMap.Value> protoEntry = headerToValueProto(
                 entry.getKey().substring(OPENSEARCH_PREFIX_KEY.length()),
                 entry.getValue()
             );
-            errorCauseBuilder.putMetadata(protoEntry.getKey(), protoEntry.getValue());
+            metadataBuilder.putFields(protoEntry.getKey(), protoEntry.getValue());
         }
 
         // Add metadata if the throwable is an OpenSearchException
         if (throwable instanceof OpenSearchException ose) {
             Map<String, ObjectMap.Value> moreMetadata = metadataToProto(ose);
             for (Map.Entry<String, ObjectMap.Value> entry : moreMetadata.entrySet()) {
-                errorCauseBuilder.putMetadata(entry.getKey(), entry.getValue());
+                metadataBuilder.putFields(entry.getKey(), entry.getValue());
             }
         }
 
+        // Set the metadata if any fields were added
+        if (metadataBuilder.getFieldsCount() > 0) {
+            errorCauseBuilder.setMetadata(metadataBuilder.build());
+        }
+
         if (cause != null) {
             errorCauseBuilder.setCausedBy(generateThrowableProto(cause));
         }