Tez 4363

BUILDING.txt

@@ -9,7 +9,7 @@ Requirements:
 * JDK 1.8+
 * Maven 3.1 or later
 * Findbugs 2.0.2 or later (if running findbugs)
-* ProtocolBuffer 2.5.0
+* ProtocolBuffer 3.19.4
 * Internet connection for first build (to fetch all dependencies)
 * Hadoop version should be 2.7.0 or higher.
 

build-tools/docker/Dockerfile

@@ -96,13 +96,13 @@ RUN add-apt-repository -y ppa:longsleep/golang-backports \
 #############
 
 ######
-# Install Google Protobuf 2.5.0
+# Install Google Protobuf 3.19.4
 ######
-FROM tezbase AS protobuf250
+FROM tezbase AS protobuf
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN mkdir -p /opt/protobuf-src \
     && curl -L -s -S \
-      https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+      https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protobuf-java-3.19.4.tar.gz \
       -o /opt/protobuf.tar.gz \
     && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
 WORKDIR /opt/protobuf-src
@@ -262,7 +262,7 @@ RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - \
 COPY --from=shellcheck /bin/shellcheck /bin/shellcheck
 COPY --from=hadolint /bin/hadolint /bin/hadolint
 COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
-COPY --from=protobuf250 /opt/protobuf /opt/protobuf
+COPY --from=protobuf /opt/protobuf /opt/protobuf
 
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PROTOC_PATH /opt/protobuf/bin/protoc

build-tools/install-protobuf.sh

@@ -25,7 +25,7 @@
 # rm -rf ./build-tools/protobuf/ #from root folder
 
 set -x
-PROTOBUF_VERSION=${1:-2.5.0}
+PROTOBUF_VERSION=${1:-3.19.4}
 PROTOBUF_MAJOR_VERSION=$(echo "$PROTOBUF_VERSION" | cut -d. -f1)
 if [ -n "$ZSH_VERSION" ]; then
    SCRIPT_DIR="${0:a:h}"

pom.xml

@@ -63,7 +63,7 @@
     <pig.version>0.13.0</pig.version>
     <jersey.version>1.19</jersey.version>
     <slf4j.version>1.7.36</slf4j.version>
-    <protobuf.version>2.5.0</protobuf.version>
+    <protobuf.version>3.19.4</protobuf.version>
     <roaringbitmap.version>0.7.45</roaringbitmap.version>
     <protoc.path>${env.PROTOC_PATH}</protoc.path>
     <scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>

tez-dag/pom.xml

@@ -206,6 +206,27 @@
           </environmentVariables>
         </configuration>
       </plugin>
+      <!-- This plugin takes care of on-the-fly installation of the needed protobuf version.
+           The needed version is always what's defined as protobuf.version in the pom,
+           so if user wants to change protobuf version quickly in development time. -->
+      <plugin>
+        <artifactId>exec-maven-plugin</artifactId>
+        <groupId>org.codehaus.mojo</groupId>
+        <version>1.6.0</version>
+        <executions>
+          <execution>
+            <id>Install protobuf</id>
+            <phase>initialize</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>${basedir}/../build-tools/install-protobuf.sh</executable>
+              <arguments>${protobuf.version}</arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-maven-plugins</artifactId>

tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.history.HistoryEvent;
@@ -90,7 +91,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    AMLaunchedProto proto = inputStream.readMessage(AMLaunchedProto.PARSER, null);
+    AMLaunchedProto proto = inputStream.readMessage(AMLaunchedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.dag.history.HistoryEvent;
@@ -85,7 +86,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    AMStartedProto proto = inputStream.readMessage(AMStartedProto.PARSER, null);
+    AMStartedProto proto = inputStream.readMessage(AMStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -83,7 +84,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    ContainerLaunchedProto proto = inputStream.readMessage(ContainerLaunchedProto.PARSER, null);
+    ContainerLaunchedProto proto =
+        inputStream.readMessage(ContainerLaunchedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -88,7 +89,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    ContainerStoppedProto proto = inputStream.readMessage(ContainerStoppedProto.PARSER, null);
+    ContainerStoppedProto proto =
+        inputStream.readMessage(ContainerStoppedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java

@@ -23,6 +23,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -76,7 +77,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGCommitStartedProto proto = inputStream.readMessage(DAGCommitStartedProto.PARSER, null);
+    DAGCommitStartedProto proto =
+        inputStream.readMessage(DAGCommitStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.DagTypeConverters;
@@ -129,7 +130,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGFinishedProto proto = inputStream.readMessage(DAGFinishedProto.PARSER, null);
+    DAGFinishedProto proto = inputStream.readMessage(DAGFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java

@@ -23,6 +23,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.records.DAGIDAware;
@@ -91,7 +92,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGInitializedProto proto = inputStream.readMessage(DAGInitializedProto.PARSER, null);
+    DAGInitializedProto proto = inputStream.readMessage(DAGInitializedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -76,7 +77,7 @@ public DAGKillRequestProto toProto() {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGKillRequestProto proto = inputStream.readMessage(DAGKillRequestProto.PARSER, null);
+    DAGKillRequestProto proto = inputStream.readMessage(DAGKillRequestProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.app.dag.DAGState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
@@ -81,7 +82,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGStartedProto proto = inputStream.readMessage(DAGStartedProto.PARSER, null);
+    DAGStartedProto proto = inputStream.readMessage(DAGStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.records.DAGIDAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -134,7 +135,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    DAGSubmittedProto proto = inputStream.readMessage(DAGSubmittedProto.PARSER, null);
+    DAGSubmittedProto proto = inputStream.readMessage(DAGSubmittedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.common.TezConverterUtils;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
@@ -235,7 +236,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskAttemptFinishedProto proto = inputStream.readMessage(TaskAttemptFinishedProto.PARSER, null);
+    TaskAttemptFinishedProto proto =
+        inputStream.readMessage(TaskAttemptFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -98,7 +99,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskAttemptStartedProto proto = inputStream.readMessage(TaskAttemptStartedProto.PARSER, null);
+    TaskAttemptStartedProto proto =
+        inputStream.readMessage(TaskAttemptStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.common.counters.CounterGroup;
 import org.apache.tez.common.counters.TezCounter;
 import org.apache.tez.dag.records.TaskIDAware;
@@ -116,7 +117,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskFinishedProto proto = inputStream.readMessage(TaskFinishedProto.PARSER, null);
+    TaskFinishedProto proto = inputStream.readMessage(TaskFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java

@@ -22,6 +22,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
@@ -90,7 +91,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    TaskStartedProto proto = inputStream.readMessage(TaskStartedProto.PARSER, null);
+    TaskStartedProto proto = inputStream.readMessage(TaskStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -80,7 +81,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexCommitStartedProto proto = inputStream.readMessage(VertexCommitStartedProto.PARSER, null);
+    VertexCommitStartedProto proto =
+        inputStream.readMessage(VertexCommitStartedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.VertexLocationHint;
@@ -162,7 +163,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexConfigurationDoneProto proto = inputStream.readMessage(VertexConfigurationDoneProto.PARSER, null);
+    VertexConfigurationDoneProto proto =
+        inputStream.readMessage(VertexConfigurationDoneProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.app.dag.impl.ServicePluginInfo;
 import org.apache.tez.dag.records.VertexIDAware;
 import org.slf4j.Logger;
@@ -131,7 +132,7 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexFinishedProto proto = inputStream.readMessage(VertexFinishedProto.PARSER, null);
+    VertexFinishedProto proto = inputStream.readMessage(VertexFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }

tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java

@@ -24,6 +24,7 @@
 
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
 import org.apache.tez.dag.history.HistoryEvent;
 import org.apache.tez.dag.history.HistoryEventType;
 import org.apache.tez.dag.history.SummaryEvent;
@@ -101,7 +102,8 @@ public void toProtoStream(CodedOutputStream outputStream) throws IOException {
 
   @Override
   public void fromProtoStream(CodedInputStream inputStream) throws IOException {
-    VertexGroupCommitFinishedProto proto = inputStream.readMessage(VertexGroupCommitFinishedProto.PARSER, null);
+    VertexGroupCommitFinishedProto proto =
+        inputStream.readMessage(VertexGroupCommitFinishedProto.PARSER, ExtensionRegistry.getEmptyRegistry());
     if (proto == null) {
       throw new IOException("No data found in stream");
     }