diff --git a/.travis.yml b/.travis.yml
index 3637a0defe..65eaf77783 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,9 +22,6 @@ sudo: required
env: MAVEN_OPTS="-Xmx2G -XX:MaxPermSize=512M"
-before_install:
- - ./build-tools/install-protobuf.sh
-
script:
- jdk_switcher use oraclejdk8
- mvn -B clean install package -DskipTests=true -Dmaven.javadoc.skip=true
diff --git a/BUILDING.txt b/BUILDING.txt
index ae81d9c6c8..8b89a559de 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -122,10 +122,11 @@ The version of Protocol Buffer compiler, protoc, can be defined on-the-fly as:
The default version is defined in the root pom.xml.
-If you have multiple versions of protoc in your system, you can set in your
-build shell the PROTOC_PATH environment variable to point to the one you
-want to use for the Tez build. If you don't define this environment variable,
-protoc is looked up in the PATH.
+If you have multiple versions of protoc in your system, you can set in your
+build shell the PROTOC_PATH environment variable to point to the one you
+want to use for the Tez build. If you don't define this environment variable then the
+embedded protoc compiler will be used with the version defined in ${protobuf.version}.
+It detects the platform and executes the corresponding protoc binary at build time.
You can also specify the path to protoc while building using -Dprotoc.path
diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile
index 3e792604ec..f51cc3bd2b 100644
--- a/build-tools/docker/Dockerfile
+++ b/build-tools/docker/Dockerfile
@@ -55,8 +55,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
rsync \
software-properties-common \
ssh-client \
- sudo \
- wget \
xz-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/build-tools/install-protobuf.sh b/build-tools/install-protobuf.sh
deleted file mode 100755
index c28729a8a7..0000000000
--- a/build-tools/install-protobuf.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script attempts to install an arbitrary version of protobuf if needed.
-# The desired version should be the first parameter: $1.
-# Typical usage: the script is automatically called from tez-api (by maven) during the build process.
-
-# This script runs from build-tools folder. The user can remove
-# the dynamically installed protobuf anytime like:
-# rm -rf ./build-tools/protobuf/ #from root folder
-
-set -x
-PROTOBUF_VERSION=${1:-2.5.0}
-PROTOBUF_MAJOR_VERSION=$(echo "$PROTOBUF_VERSION" | cut -d. -f1)
-if [ -n "$ZSH_VERSION" ]; then
- SCRIPT_DIR="${0:a:h}"
-else
- SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-fi
-
-function install_protobuf {
- # before protobuf 3, there is no pre-compiled executables are host on github, let's try to build and make it
- if (( PROTOBUF_MAJOR_VERSION < 3 )); then
- wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz"
- tar -xzvf "protobuf-$PROTOBUF_VERSION.tar.gz"
- rm "protobuf-$PROTOBUF_VERSION.tar.gz"
- cd "protobuf-$PROTOBUF_VERSION" && ./configure --prefix=/usr && make && sudo make install
- # since protobuf 3, there are precompiled protoc executables on github, let's quickly download and use it
- else
- ARCH=`uname -m`
- case "$(uname -s)" in
- Darwin)
- FILE_NAME="protoc-$PROTOBUF_VERSION-osx-$ARCH"
- ;;
- Linux)
- if test $ARCH = "aarch64"; then
- ARCH="aarch_64"
- fi
- FILE_NAME="protoc-$PROTOBUF_VERSION-linux-$ARCH"
- ;;
- *)
- echo "Unsupported OS returned by uname -s, you'll have to install protobuf 3.x manually"
- exit 1
- ;;
- esac
- rm -f "$FILE_NAME.zip" #cleanup unfinished file if any
- wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/$FILE_NAME.zip"
- mkdir "$SCRIPT_DIR/protobuf"
- unzip -o "$FILE_NAME.zip" -d "$SCRIPT_DIR/protobuf"
- rm "$FILE_NAME.zip"
- fi
-}
-
-if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
- PROTOBUF_INSTALLED_VERSION=$("$SCRIPT_DIR/protobuf/bin/protoc" --version)
-else
- PROTOBUF_INSTALLED_VERSION=$(protoc --version)
-fi
-
-PROTOC_EXIT_CODE=$?
-
-if [ $PROTOC_EXIT_CODE -eq 0 ]; then
- PROTOBUF_INSTALLED_VERSION=$(echo "$PROTOBUF_INSTALLED_VERSION" | tr -s ' ' | cut -d ' ' -f 2)
- if [ "$PROTOBUF_INSTALLED_VERSION" == "$PROTOBUF_VERSION" ]; then
- echo "Current protobuf version is equal to the requested ($PROTOBUF_INSTALLED_VERSION), exiting..."
- else
- echo "Current protobuf version ($PROTOBUF_INSTALLED_VERSION) is not equal to the requested ($PROTOBUF_VERSION), installing $PROTOBUF_VERSION"
- install_protobuf
- fi
-else
- echo "protoc --version command had non-zero return value, need to install probuf"
- install_protobuf
-fi
diff --git a/build-tools/protocw b/build-tools/protocw
deleted file mode 100755
index 6196071815..0000000000
--- a/build-tools/protocw
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### This is a protoc wrapper for tez, which can dinamically call protoc from a downloaded protobuf.
-
-if [ -n "$ZSH_VERSION" ]; then
- SCRIPT_DIR="${0:a:h}"
-else
- SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-fi
-
-if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
- "$SCRIPT_DIR/protobuf/bin/protoc" "$@"
-else
- protoc "$@"
-fi
-exit $?
diff --git a/pom.xml b/pom.xml
index c496e0ce87..0d5de4e0dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1310,28 +1310,6 @@
-
-
- protoc-path-env-variable-not-defined
-
-
- !env.PROTOC_PATH
-
-
-
- ${basedir}/../build-tools/protocw
-
-
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index cd5abdb03a..1d10bc1f85 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -145,53 +145,28 @@
org.apache.ratapache-rat-plugin
-
- exec-maven-plugin
- org.codehaus.mojo
- 1.6.0
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- Install protobuf
- initialize
-
- exec
-
-
- ${basedir}/../build-tools/install-protobuf.sh
- ${protobuf.version}
-
-
-
-
-
- org.apache.hadoop
- hadoop-maven-plugins
-
-
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- DAGApiRecords.proto
- DAGClientAMProtocol.proto
- Events.proto
-
-
-
+ none
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 356ed2c2e9..fda9fb2a54 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -211,30 +211,31 @@
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
- ${basedir}/../tez-api/src/main/proto
- ${basedir}/../tez-runtime-internals/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- HistoryEvents.proto
-
-
-
+ none
+
+ ${basedir}/../tez-api/src/main/proto
+ ${basedir}/../tez-runtime-internals/src/main/proto
+
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
index 609b896fd8..a0bcbe32c0 100644
--- a/tez-ext-service-tests/pom.xml
+++ b/tez-ext-service-tests/pom.xml
@@ -165,29 +165,30 @@
apache-rat-plugin
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/test/proto
+ none
+
${basedir}/../tez-api/src/main/proto
-
-
- ${basedir}/src/test/proto
-
- TezDaemonProtocol.proto
-
-
-
+
+
+ ${basedir}/src/test/proto
+
+
+
+ ${project.build.directory}/generated-test-sources/java
+
+
diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml
index a22870fc38..1e35d2ea11 100644
--- a/tez-mapreduce/pom.xml
+++ b/tez-mapreduce/pom.xml
@@ -137,28 +137,27 @@
apache-rat-plugin
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- MRRuntimeProtos.proto
-
-
-
+ none
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+
diff --git a/tez-plugins/tez-protobuf-history-plugin/pom.xml b/tez-plugins/tez-protobuf-history-plugin/pom.xml
index 607a31f95b..0fa526413e 100644
--- a/tez-plugins/tez-protobuf-history-plugin/pom.xml
+++ b/tez-plugins/tez-protobuf-history-plugin/pom.xml
@@ -59,47 +59,31 @@
apache-rat-plugin
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- HistoryLogger.proto
-
-
-
+ none
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+
-
-
-
-
- protoc-path-env-variable-not-defined
-
-
- !env.PROTOC_PATH
-
-
-
- ${basedir}/../../build-tools/protocw
-
-
-
diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml
index 0619ca1e16..efe01e058e 100644
--- a/tez-runtime-internals/pom.xml
+++ b/tez-runtime-internals/pom.xml
@@ -91,28 +91,27 @@
apache-rat-plugin
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- RuntimeEvents.proto
-
-
-
+ none
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+
diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml
index c913e87d43..e6db47e535 100644
--- a/tez-runtime-library/pom.xml
+++ b/tez-runtime-library/pom.xml
@@ -114,30 +114,27 @@
apache-rat-plugin
- org.apache.hadoop
- hadoop-maven-plugins
+ com.github.os72
+ protoc-jar-maven-plugin
+ 3.11.4
- compile-protocgenerate-sources
- protoc
+ run
- ${protobuf.version}
+ com.google.protobuf:protoc:${protobuf.version}${protoc.path}
-
- ${basedir}/src/main/proto
-
-
- ${basedir}/src/main/proto
-
- ShufflePayloads.proto
- CartesianProductPayload.proto
- FairShufflePayloads.proto
-
-
-
+ none
+
+ ${basedir}/src/main/proto
+
+
+
+ ${project.build.directory}/generated-sources/java
+
+