diff --git a/dev-support/bin/tez_run_example.sh b/dev-support/bin/tez_run_example.sh index 7e980b9484..144b6485c2 100755 --- a/dev-support/bin/tez_run_example.sh +++ b/dev-support/bin/tez_run_example.sh @@ -1,3 +1,19 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # This script is used to set up a local Hadoop and Tez environment for running a simple word count example. # Prerequisites @@ -10,19 +26,19 @@ # TEZ_EXAMPLE_WORKING_DIR: defaults to the current working directory # TEZ_VERSION comes from environment variable or is fetched from the Apache Tez download page -export TEZ_VERSION=${TEZ_VERSION:=$(curl -s "https://downloads.apache.org/tez/" | grep --color=never -o '[0-9]\+\.[0-9]\+\.[0-9]\+' | sed -n '/\/$/!p' | sort -V | tail -1)} # e.g. 0.10.4 -export TEZ_EXAMPLE_WORKING_DIR=${TEZ_EXAMPLE_WORKING_DIR:=$PWD} -cd $TEZ_EXAMPLE_WORKING_DIR +export TEZ_VERSION="${TEZ_VERSION:-$(curl -s "https://downloads.apache.org/tez/" | grep --color=never -o '[0-9]\+\.[0-9]\+\.[0-9]\+' | sed -n '/\/$/!p' | sort -V | tail -1)}" # e.g. 0.10.4 +export TEZ_EXAMPLE_WORKING_DIR="${TEZ_EXAMPLE_WORKING_DIR:-$PWD}" +cd "$TEZ_EXAMPLE_WORKING_DIR" || exit echo "TEZ_VERSION: $TEZ_VERSION" -wget -nc https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz +wget -nc "https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz" # Need to extract the Tez tarball early to get hadoop version it depends on if [ ! -d "apache-tez-$TEZ_VERSION-bin" ]; then - tar -xzf apache-tez-$TEZ_VERSION-bin.tar.gz + tar -xzf "apache-tez-$TEZ_VERSION-bin.tar.gz" fi -export HADOOP_VERSION=${HADOOP_VERSION:=$(basename apache-tez-$TEZ_VERSION-bin/lib/hadoop-hdfs-client-*.jar | sed -E 's/.*hadoop-hdfs-client-([0-9]+\.[0-9]+\.[0-9]+)\.jar/\1/')} # e.g. 3.4.1 +export HADOOP_VERSION="${HADOOP_VERSION:-$(basename "apache-tez-$TEZ_VERSION-bin/lib/hadoop-hdfs-client-"*.jar | sed -E 's/.*hadoop-hdfs-client-([0-9]+\.[0-9]+\.[0-9]+)\.jar/\1/')}" # e.g. 3.4.1 cat < $HADOOP_HOME/etc/hadoop/hdfs-site.xml +cat < "$HADOOP_HOME/etc/hadoop/hdfs-site.xml" @@ -57,7 +73,7 @@ cat < $HADOOP_HOME/etc/hadoop/hdfs-site.xml EOF -cat < $HADOOP_HOME/etc/hadoop/core-site.xml +cat < "$HADOOP_HOME/etc/hadoop/core-site.xml" @@ -69,7 +85,7 @@ cat < $HADOOP_HOME/etc/hadoop/core-site.xml EOF -cat < $HADOOP_HOME/etc/hadoop/yarn-site.xml +cat < "$HADOOP_HOME/etc/hadoop/yarn-site.xml" @@ -82,20 +98,20 @@ cat < $HADOOP_HOME/etc/hadoop/yarn-site.xml EOF # optionally stop previous clusters if any -$HADOOP_HOME/sbin/stop-dfs.sh -$HADOOP_HOME/sbin/stop-yarn.sh +"$HADOOP_HOME/sbin/stop-dfs.sh" +"$HADOOP_HOME/sbin/stop-yarn.sh" -rm -rf /tmp/hadoop-$USER/dfs/data +rm -rf "/tmp/hadoop-$USER/dfs/data" hdfs namenode -format -force -$HADOOP_HOME/sbin/start-dfs.sh -$HADOOP_HOME/sbin/start-yarn.sh +"$HADOOP_HOME/sbin/start-dfs.sh" +"$HADOOP_HOME/sbin/start-yarn.sh" -hadoop fs -mkdir -p /apps/tez-$TEZ_VERSION -hadoop fs -copyFromLocal $TEZ_HOME/share/tez.tar.gz /apps/tez-$TEZ_VERSION +hadoop fs -mkdir -p "/apps/tez-$TEZ_VERSION" +hadoop fs -copyFromLocal "$TEZ_HOME/share/tez.tar.gz" "/apps/tez-$TEZ_VERSION" # create a simple tez-site.xml -cat < $TEZ_HOME/conf/tez-site.xml +cat < "$TEZ_HOME/conf/tez-site.xml" @@ -121,24 +137,23 @@ Friend Game EOF -hadoop fs -copyFromLocal words.txt /words.txt +hadoop fs -copyFromLocal "words.txt" "/words.txt" export HADOOP_USER_CLASSPATH_FIRST=true # finally run the example -yarn jar $TEZ_HOME/tez-examples-$TEZ_VERSION.jar orderedwordcount /words.txt /words_out +yarn jar "$TEZ_HOME/tez-examples-$TEZ_VERSION.jar" orderedwordcount "/words.txt" "/words_out" # check the output -hadoop fs -ls /words_out -hadoop fs -text /words_out/part-v002-o000-r-00000 - +hadoop fs -ls "/words_out" +hadoop fs -text "/words_out/part-v002-o000-r-00000" cat <