diff --git a/.travis.yml b/.travis.yml index 3637a0defe..65eaf77783 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,6 @@ sudo: required env: MAVEN_OPTS="-Xmx2G -XX:MaxPermSize=512M" -before_install: - - ./build-tools/install-protobuf.sh - script: - jdk_switcher use oraclejdk8 - mvn -B clean install package -DskipTests=true -Dmaven.javadoc.skip=true diff --git a/BUILDING.txt b/BUILDING.txt index ae81d9c6c8..8b89a559de 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -122,10 +122,11 @@ The version of Protocol Buffer compiler, protoc, can be defined on-the-fly as: The default version is defined in the root pom.xml. -If you have multiple versions of protoc in your system, you can set in your -build shell the PROTOC_PATH environment variable to point to the one you -want to use for the Tez build. If you don't define this environment variable, -protoc is looked up in the PATH. +If you have multiple versions of protoc in your system, you can set in your +build shell the PROTOC_PATH environment variable to point to the one you +want to use for the Tez build. If you don't define this environment variable then the +embedded protoc compiler will be used with the version defined in ${protobuf.version}. +It detects the platform and executes the corresponding protoc binary at build time. You can also specify the path to protoc while building using -Dprotoc.path diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 3e792604ec..f51cc3bd2b 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -55,8 +55,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ rsync \ software-properties-common \ ssh-client \ - sudo \ - wget \ xz-utils \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/build-tools/install-protobuf.sh b/build-tools/install-protobuf.sh deleted file mode 100755 index c28729a8a7..0000000000 --- a/build-tools/install-protobuf.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script attempts to install an arbitrary version of protobuf if needed. -# The desired version should be the first parameter: $1. -# Typical usage: the script is automatically called from tez-api (by maven) during the build process. - -# This script runs from build-tools folder. The user can remove -# the dynamically installed protobuf anytime like: -# rm -rf ./build-tools/protobuf/ #from root folder - -set -x -PROTOBUF_VERSION=${1:-2.5.0} -PROTOBUF_MAJOR_VERSION=$(echo "$PROTOBUF_VERSION" | cut -d. -f1) -if [ -n "$ZSH_VERSION" ]; then - SCRIPT_DIR="${0:a:h}" -else - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -fi - -function install_protobuf { - # before protobuf 3, there is no pre-compiled executables are host on github, let's try to build and make it - if (( PROTOBUF_MAJOR_VERSION < 3 )); then - wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz" - tar -xzvf "protobuf-$PROTOBUF_VERSION.tar.gz" - rm "protobuf-$PROTOBUF_VERSION.tar.gz" - cd "protobuf-$PROTOBUF_VERSION" && ./configure --prefix=/usr && make && sudo make install - # since protobuf 3, there are precompiled protoc executables on github, let's quickly download and use it - else - ARCH=`uname -m` - case "$(uname -s)" in - Darwin) - FILE_NAME="protoc-$PROTOBUF_VERSION-osx-$ARCH" - ;; - Linux) - if test $ARCH = "aarch64"; then - ARCH="aarch_64" - fi - FILE_NAME="protoc-$PROTOBUF_VERSION-linux-$ARCH" - ;; - *) - echo "Unsupported OS returned by uname -s, you'll have to install protobuf 3.x manually" - exit 1 - ;; - esac - rm -f "$FILE_NAME.zip" #cleanup unfinished file if any - wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/$FILE_NAME.zip" - mkdir "$SCRIPT_DIR/protobuf" - unzip -o "$FILE_NAME.zip" -d "$SCRIPT_DIR/protobuf" - rm "$FILE_NAME.zip" - fi -} - -if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then - PROTOBUF_INSTALLED_VERSION=$("$SCRIPT_DIR/protobuf/bin/protoc" --version) -else - PROTOBUF_INSTALLED_VERSION=$(protoc --version) -fi - -PROTOC_EXIT_CODE=$? - -if [ $PROTOC_EXIT_CODE -eq 0 ]; then - PROTOBUF_INSTALLED_VERSION=$(echo "$PROTOBUF_INSTALLED_VERSION" | tr -s ' ' | cut -d ' ' -f 2) - if [ "$PROTOBUF_INSTALLED_VERSION" == "$PROTOBUF_VERSION" ]; then - echo "Current protobuf version is equal to the requested ($PROTOBUF_INSTALLED_VERSION), exiting..." - else - echo "Current protobuf version ($PROTOBUF_INSTALLED_VERSION) is not equal to the requested ($PROTOBUF_VERSION), installing $PROTOBUF_VERSION" - install_protobuf - fi -else - echo "protoc --version command had non-zero return value, need to install probuf" - install_protobuf -fi diff --git a/build-tools/protocw b/build-tools/protocw deleted file mode 100755 index 6196071815..0000000000 --- a/build-tools/protocw +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -### This is a protoc wrapper for tez, which can dinamically call protoc from a downloaded protobuf. - -if [ -n "$ZSH_VERSION" ]; then - SCRIPT_DIR="${0:a:h}" -else - SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -fi - -if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then - "$SCRIPT_DIR/protobuf/bin/protoc" "$@" -else - protoc "$@" -fi -exit $? diff --git a/pom.xml b/pom.xml index c496e0ce87..0d5de4e0dc 100644 --- a/pom.xml +++ b/pom.xml @@ -1310,28 +1310,6 @@ - - - protoc-path-env-variable-not-defined - - - !env.PROTOC_PATH - - - - ${basedir}/../build-tools/protocw - - diff --git a/tez-api/pom.xml b/tez-api/pom.xml index cd5abdb03a..1d10bc1f85 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -145,53 +145,28 @@ org.apache.rat apache-rat-plugin - - exec-maven-plugin - org.codehaus.mojo - 1.6.0 + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - Install protobuf - initialize - - exec - - - ${basedir}/../build-tools/install-protobuf.sh - ${protobuf.version} - - - - - - org.apache.hadoop - hadoop-maven-plugins - - - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - DAGApiRecords.proto - DAGClientAMProtocol.proto - Events.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + + diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 356ed2c2e9..fda9fb2a54 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -211,30 +211,31 @@ - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - ${basedir}/../tez-api/src/main/proto - ${basedir}/../tez-runtime-internals/src/main/proto - - - ${basedir}/src/main/proto - - HistoryEvents.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/../tez-api/src/main/proto + ${basedir}/../tez-runtime-internals/src/main/proto + + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + + diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index 609b896fd8..a0bcbe32c0 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -165,29 +165,30 @@ apache-rat-plugin - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/test/proto + none + ${basedir}/../tez-api/src/main/proto - - - ${basedir}/src/test/proto - - TezDaemonProtocol.proto - - - ${project.build.directory}/generated-test-sources/java + + + ${basedir}/src/test/proto + + + + ${project.build.directory}/generated-test-sources/java + + diff --git a/tez-mapreduce/pom.xml b/tez-mapreduce/pom.xml index a22870fc38..1e35d2ea11 100644 --- a/tez-mapreduce/pom.xml +++ b/tez-mapreduce/pom.xml @@ -137,28 +137,27 @@ apache-rat-plugin - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - MRRuntimeProtos.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + + diff --git a/tez-plugins/tez-protobuf-history-plugin/pom.xml b/tez-plugins/tez-protobuf-history-plugin/pom.xml index 607a31f95b..0fa526413e 100644 --- a/tez-plugins/tez-protobuf-history-plugin/pom.xml +++ b/tez-plugins/tez-protobuf-history-plugin/pom.xml @@ -59,47 +59,31 @@ apache-rat-plugin - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - HistoryLogger.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + + - - - - - protoc-path-env-variable-not-defined - - - !env.PROTOC_PATH - - - - ${basedir}/../../build-tools/protocw - - - diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index 0619ca1e16..efe01e058e 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -91,28 +91,27 @@ apache-rat-plugin - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - RuntimeEvents.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + + diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml index c913e87d43..e6db47e535 100644 --- a/tez-runtime-library/pom.xml +++ b/tez-runtime-library/pom.xml @@ -114,30 +114,27 @@ apache-rat-plugin - org.apache.hadoop - hadoop-maven-plugins + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 - compile-protoc generate-sources - protoc + run - ${protobuf.version} + com.google.protobuf:protoc:${protobuf.version} ${protoc.path} - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - ShufflePayloads.proto - CartesianProductPayload.proto - FairShufflePayloads.proto - - - ${project.build.directory}/generated-sources/java + none + + ${basedir}/src/main/proto + + + + ${project.build.directory}/generated-sources/java + +