TEZ-4363: Bump protobuf dependency to 3.x

BUILDING.txt

@@ -9,7 +9,7 @@ Requirements:
 * JDK 1.8+
 * Maven 3.1 or later
 * Findbugs 2.0.2 or later (if running findbugs)
-* ProtocolBuffer 2.5.0
+* ProtocolBuffer 3.21.1
 * Internet connection for first build (to fetch all dependencies)
 * Hadoop version should be 2.7.0 or higher.
 

build-tools/docker/Dockerfile

@@ -95,13 +95,13 @@ RUN add-apt-repository -y ppa:longsleep/golang-backports \
 #############
 
 ######
-# Install Google Protobuf 2.5.0
+# Install Google Protobuf 3.21.1
 ######
-FROM tezbase AS protobuf250
+FROM tezbase AS protobuf
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN mkdir -p /opt/protobuf-src \
     && curl -L -s -S \
-      https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+      https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protobuf-java-3.21.1.tar.gz \
       -o /opt/protobuf.tar.gz \
     && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
 WORKDIR /opt/protobuf-src
@@ -261,7 +261,7 @@ RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - \
 COPY --from=shellcheck /bin/shellcheck /bin/shellcheck
 COPY --from=hadolint /bin/hadolint /bin/hadolint
 COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
-COPY --from=protobuf250 /opt/protobuf /opt/protobuf
+COPY --from=protobuf /opt/protobuf /opt/protobuf
 
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PROTOC_PATH /opt/protobuf/bin/protoc

pom.xml

@@ -63,7 +63,7 @@
     <pig.version>0.13.0</pig.version>
     <jersey.version>1.19</jersey.version>
     <slf4j.version>1.7.36</slf4j.version>
-    <protobuf.version>2.5.0</protobuf.version>
+    <protobuf.version>3.21.1</protobuf.version>
     <roaringbitmap.version>0.7.45</roaringbitmap.version>
     <protoc.path>${env.PROTOC_PATH}</protoc.path>
     <scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>

tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.history.HistoryEvent;
@@ -90,7 +91,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    AMLaunchedProto proto = inputStream.readMessage(AMLaunchedProto.PARSER, null);
+    AMLaunchedProto proto = inputStream.readMessage(AMLaunchedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.history.HistoryEvent;
@@ -85,7 +86,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    AMStartedProto proto = inputStream.readMessage(AMStartedProto.PARSER, null);
+    AMStartedProto proto = inputStream.readMessage(AMStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -83,7 +84,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    ContainerLaunchedProto proto = inputStream.readMessage(ContainerLaunchedProto.PARSER, null);
+    ContainerLaunchedProto proto =
+        inputStream.readMessage(ContainerLaunchedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -88,7 +89,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    ContainerStoppedProto proto = inputStream.readMessage(ContainerStoppedProto.PARSER, null);
+    ContainerStoppedProto proto =
+        inputStream.readMessage(ContainerStoppedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java

@@ -23,6 +23,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -76,7 +77,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGCommitStartedProto proto = inputStream.readMessage(DAGCommitStartedProto.PARSER, null);
+    DAGCommitStartedProto proto =
+        inputStream.readMessage(DAGCommitStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.DagTypeConverters;
@@ -129,7 +130,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGFinishedProto proto = inputStream.readMessage(DAGFinishedProto.PARSER, null);
+    DAGFinishedProto proto = inputStream.readMessage(DAGFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java

@@ -23,6 +23,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.records.DAGIDAware;
@@ -91,7 +92,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGInitializedProto proto = inputStream.readMessage(DAGInitializedProto.PARSER, null);
+    DAGInitializedProto proto =
+        inputStream.readMessage(DAGInitializedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -76,7 +77,8 @@ public DAGKillRequestProto toProto() {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGKillRequestProto proto = inputStream.readMessage(DAGKillRequestProto.PARSER, null);
+    DAGKillRequestProto proto =
+        inputStream.readMessage(DAGKillRequestProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.app.dag.DAGState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
@@ -81,7 +82,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGStartedProto proto = inputStream.readMessage(DAGStartedProto.PARSER, null);
+    DAGStartedProto proto = inputStream.readMessage(DAGStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.records.DAGIDAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -134,7 +135,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGSubmittedProto proto = inputStream.readMessage(DAGSubmittedProto.PARSER, null);
+    DAGSubmittedProto proto = inputStream.readMessage(DAGSubmittedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.common.TezConverterUtils;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
@@ -235,7 +236,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskAttemptFinishedProto proto = inputStream.readMessage(TaskAttemptFinishedProto.PARSER, null);
+    TaskAttemptFinishedProto proto =
+        inputStream.readMessage(TaskAttemptFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -98,7 +99,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskAttemptStartedProto proto = inputStream.readMessage(TaskAttemptStartedProto.PARSER, null);
+    TaskAttemptStartedProto proto =
+        inputStream.readMessage(TaskAttemptStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.dag.records.TaskIDAware;
@@ -116,7 +117,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskFinishedProto proto = inputStream.readMessage(TaskFinishedProto.PARSER, null);
+    TaskFinishedProto proto = inputStream.readMessage(TaskFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
@@ -90,7 +91,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskStartedProto proto = inputStream.readMessage(TaskStartedProto.PARSER, null);
+    TaskStartedProto proto = inputStream.readMessage(TaskStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -80,7 +81,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexCommitStartedProto proto = inputStream.readMessage(VertexCommitStartedProto.PARSER, null);
+    VertexCommitStartedProto proto =
+        inputStream.readMessage(VertexCommitStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.VertexLocationHint;
@@ -162,7 +163,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexConfigurationDoneProto proto = inputStream.readMessage(VertexConfigurationDoneProto.PARSER, null);
+    VertexConfigurationDoneProto proto =
+        inputStream.readMessage(VertexConfigurationDoneProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.app.dag.impl.ServicePluginInfo;
 import org.apache.tez.dag.records.VertexIDAware;
 import org.slf4j.Logger;
@@ -131,7 +132,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexFinishedProto proto = inputStream.readMessage(VertexFinishedProto.PARSER, null);
+    VertexFinishedProto proto =
+        inputStream.readMessage(VertexFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -101,7 +102,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexGroupCommitFinishedProto proto = inputStream.readMessage(VertexGroupCommitFinishedProto.PARSER, null);
+    VertexGroupCommitFinishedProto proto =
+        inputStream.readMessage(VertexGroupCommitFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -101,7 +102,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexGroupCommitStartedProto proto = inputStream.readMessage(VertexGroupCommitStartedProto.PARSER, null);
+    VertexGroupCommitStartedProto proto =
+        inputStream.readMessage(VertexGroupCommitStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java

@@ -25,6 +25,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.InputInitializerDescriptor;
@@ -158,7 +159,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexInitializedProto proto = inputStream.readMessage(VertexInitializedProto.PARSER, null);
+    VertexInitializedProto proto =
+        inputStream.readMessage(VertexInitializedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }