Skip to content

Commit 52a1cd2

Browse files
sarutaksarutak
committed
Fixed comflicts
2 parents f1c90bb + c6e0c2a commit 52a1cd2

File tree

676 files changed

+35259
-6747
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

676 files changed

+35259
-6747
lines changed

‎.rat-excludes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ SparkImports.scala
4444
SparkJLineCompletion.scala
4545
SparkJLineReader.scala
4646
SparkMemberHandlers.scala
47+
SparkReplReporter.scala
4748
sbt
4849
sbt-launch-lib.bash
4950
plugins.sbt

‎LICENSE‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -754,7 +754,7 @@ SUCH DAMAGE.
754754

755755

756756
========================================================================
757-
For Timsort (core/src/main/java/org/apache/spark/util/collection/Sorter.java):
757+
For Timsort (core/src/main/java/org/apache/spark/util/collection/TimSort.java):
758758
========================================================================
759759
Copyright (C) 2008 The Android Open Source Project
760760

@@ -771,6 +771,25 @@ See the License for the specific language governing permissions and
771771
limitations under the License.
772772

773773

774+
========================================================================
775+
For LimitedInputStream
776+
(network/common/src/main/java/org/apache/spark/network/util/LimitedInputStream.java):
777+
========================================================================
778+
Copyright (C) 2007 The Guava Authors
779+
780+
Licensed under the Apache License, Version 2.0 (the "License");
781+
you may not use this file except in compliance with the License.
782+
You may obtain a copy of the License at
783+
784+
http://www.apache.org/licenses/LICENSE-2.0
785+
786+
Unless required by applicable law or agreed to in writing, software
787+
distributed under the License is distributed on an "AS IS" BASIS,
788+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
789+
See the License for the specific language governing permissions and
790+
limitations under the License.
791+
792+
774793
========================================================================
775794
BSD-style licenses
776795
========================================================================

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ and Spark Streaming for stream processing.
1313
## Online Documentation
1414

1515
You can find the latest Spark documentation, including a programming
16-
guide, on the [project web page](http://spark.apache.org/documentation.html).
16+
guide, on the [project web page](http://spark.apache.org/documentation.html)
17+
and [project wiki](https://cwiki.apache.org/confluence/display/SPARK).
1718
This README file only contains basic setup instructions.
1819

1920
## Building Spark

‎assembly/pom.xml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,22 @@
6666
</dependency>
6767
<dependency>
6868
<groupId>org.apache.spark</groupId>
69-
<artifactId>spark-repl_${scala.binary.version}</artifactId>
69+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
7070
<version>${project.version}</version>
7171
</dependency>
7272
<dependency>
7373
<groupId>org.apache.spark</groupId>
74-
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
74+
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
7575
<version>${project.version}</version>
7676
</dependency>
7777
<dependency>
7878
<groupId>org.apache.spark</groupId>
79-
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
79+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
8080
<version>${project.version}</version>
8181
</dependency>
8282
<dependency>
8383
<groupId>org.apache.spark</groupId>
84-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
84+
<artifactId>spark-repl_${scala.binary.version}</artifactId>
8585
<version>${project.version}</version>
8686
</dependency>
8787
</dependencies>
@@ -146,10 +146,6 @@
146146
<exclude>com/google/common/base/Present*</exclude>
147147
</excludes>
148148
</relocation>
149-
<relocation>
150-
<pattern>org.apache.commons.math3</pattern>
151-
<shadedPattern>org.spark-project.commons.math3</shadedPattern>
152-
</relocation>
153149
</relocations>
154150
<transformers>
155151
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
@@ -201,6 +197,11 @@
201197
<artifactId>spark-hive_${scala.binary.version}</artifactId>
202198
<version>${project.version}</version>
203199
</dependency>
200+
</dependencies>
201+
</profile>
202+
<profile>
203+
<id>hive-thriftserver</id>
204+
<dependencies>
204205
<dependency>
205206
<groupId>org.apache.spark</groupId>
206207
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>

‎bin/compute-classpath.sh‎

Lines changed: 22 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
# This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
2121
# script and the ExecutorRunner in standalone cluster mode.
2222

23-
SCALA_VERSION=2.10
24-
2523
# Figure out where Spark is installed
2624
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2725

@@ -36,7 +34,7 @@ else
3634
CLASSPATH="$CLASSPATH:$FWDIR/conf"
3735
fi
3836

39-
ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION"
37+
ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SPARK_SCALA_VERSION"
4038

4139
if [ -n "$JAVA_HOME" ]; then
4240
JAR_CMD="$JAVA_HOME/bin/jar"
@@ -48,19 +46,19 @@ fi
4846
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
4947
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
5048
"classes ahead of assembly." >&2
51-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/classes"
49+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
5250
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
53-
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SCALA_VERSION/classes"
54-
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SCALA_VERSION/classes"
55-
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SCALA_VERSION/classes"
56-
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SCALA_VERSION/classes"
57-
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SCALA_VERSION/classes"
58-
CLASSPATH="$CLASSPATH:$FWDIR/tools/target/scala-$SCALA_VERSION/classes"
59-
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes"
60-
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes"
61-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes"
62-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SCALA_VERSION/classes"
63-
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes"
51+
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
52+
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
53+
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
54+
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SPARK_SCALA_VERSION/classes"
55+
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SPARK_SCALA_VERSION/classes"
56+
CLASSPATH="$CLASSPATH:$FWDIR/tools/target/scala-$SPARK_SCALA_VERSION/classes"
57+
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SPARK_SCALA_VERSION/classes"
58+
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SPARK_SCALA_VERSION/classes"
59+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
60+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
61+
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
6462
fi
6563

6664
# Use spark-assembly jar from either RELEASE or assembly directory
@@ -123,15 +121,15 @@ fi
123121

124122
# Add test classes if we're running from SBT or Maven with SPARK_TESTING set to 1
125123
if [[ $SPARK_TESTING == 1 ]]; then
126-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/test-classes"
127-
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SCALA_VERSION/test-classes"
128-
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SCALA_VERSION/test-classes"
129-
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SCALA_VERSION/test-classes"
130-
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SCALA_VERSION/test-classes"
131-
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SCALA_VERSION/test-classes"
132-
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/test-classes"
133-
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/test-classes"
134-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/test-classes"
124+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
125+
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/test-classes"
126+
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/test-classes"
127+
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/test-classes"
128+
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SPARK_SCALA_VERSION/test-classes"
129+
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SPARK_SCALA_VERSION/test-classes"
130+
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SPARK_SCALA_VERSION/test-classes"
131+
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
132+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/test-classes"
135133
fi
136134

137135
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail !

‎bin/load-spark-env.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,23 @@ if [ -z "$SPARK_ENV_LOADED" ]; then
3636
set +a
3737
fi
3838
fi
39+
40+
# Setting SPARK_SCALA_VERSION if not already set.
41+
42+
if [ -z "$SPARK_SCALA_VERSION" ]; then
43+
44+
ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11"
45+
ASSEMBLY_DIR1="$FWDIR/assembly/target/scala-2.10"
46+
47+
if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
48+
echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
49+
echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
50+
exit 1
51+
fi
52+
53+
if [ -d "$ASSEMBLY_DIR2" ]; then
54+
export SPARK_SCALA_VERSION="2.11"
55+
else
56+
export SPARK_SCALA_VERSION="2.10"
57+
fi
58+
fi

‎bin/pyspark‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ export SPARK_HOME="$FWDIR"
2525

2626
source "$FWDIR/bin/utils.sh"
2727

28-
SCALA_VERSION=2.10
28+
source "$FWDIR"/bin/load-spark-env.sh
2929

3030
function usage() {
3131
echo "Usage: ./bin/pyspark [options]" 1>&2
@@ -40,16 +40,14 @@ fi
4040
# Exit if the user hasn't compiled Spark
4141
if [ ! -f "$FWDIR/RELEASE" ]; then
4242
# Exit if the user hasn't compiled Spark
43-
ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
43+
ls "$FWDIR"/assembly/target/scala-$SPARK_SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
4444
if [[ $? != 0 ]]; then
4545
echo "Failed to find Spark assembly in $FWDIR/assembly/target" 1>&2
4646
echo "You need to build Spark before running this program" 1>&2
4747
exit 1
4848
fi
4949
fi
5050

51-
. "$FWDIR"/bin/load-spark-env.sh
52-
5351
# In Spark <= 1.1, setting IPYTHON=1 would cause the driver to be launched using the `ipython`
5452
# executable, while the worker would still be launched using PYSPARK_PYTHON.
5553
#
@@ -134,7 +132,5 @@ if [[ "$1" =~ \.py$ ]]; then
134132
gatherSparkSubmitOpts "$@"
135133
exec "$FWDIR"/bin/spark-submit "${SUBMISSION_OPTS[@]}" "$primary" "${APPLICATION_OPTS[@]}"
136134
else
137-
# PySpark shell requires special handling downstream
138-
export PYSPARK_SHELL=1
139135
exec "$PYSPARK_DRIVER_PYTHON" $PYSPARK_DRIVER_PYTHON_OPTS
140136
fi

‎bin/pyspark2.cmd‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ for /f %%i in ('echo %1^| findstr /R "\.py"') do (
5959
)
6060

6161
if [%PYTHON_FILE%] == [] (
62-
set PYSPARK_SHELL=1
6362
if [%IPYTHON%] == [1] (
6463
ipython %IPYTHON_OPTS%
6564
) else (

‎bin/run-example‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@
1717
# limitations under the License.
1818
#
1919

20-
SCALA_VERSION=2.10
21-
2220
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2321
export SPARK_HOME="$FWDIR"
2422
EXAMPLES_DIR="$FWDIR"/examples
2523

24+
. "$FWDIR"/bin/load-spark-env.sh
25+
2626
if [ -n "$1" ]; then
2727
EXAMPLE_CLASS="$1"
2828
shift
@@ -36,8 +36,8 @@ fi
3636

3737
if [ -f "$FWDIR/RELEASE" ]; then
3838
export SPARK_EXAMPLES_JAR="`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`"
39-
elif [ -e "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
40-
export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar`"
39+
elif [ -e "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
40+
export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar`"
4141
fi
4242

4343
if [[ -z "$SPARK_EXAMPLES_JAR" ]]; then

‎bin/spark-class‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ case "`uname`" in
2424
CYGWIN*) cygwin=true;;
2525
esac
2626

27-
SCALA_VERSION=2.10
28-
2927
# Figure out where Spark is installed
3028
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
3129

@@ -128,9 +126,9 @@ fi
128126

129127
TOOLS_DIR="$FWDIR"/tools
130128
SPARK_TOOLS_JAR=""
131-
if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar ]; then
129+
if [ -e "$TOOLS_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-tools*[0-9Tg].jar ]; then
132130
# Use the JAR from the SBT build
133-
export SPARK_TOOLS_JAR="`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar`"
131+
export SPARK_TOOLS_JAR="`ls "$TOOLS_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-tools*[0-9Tg].jar`"
134132
fi
135133
if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then
136134
# Use the JAR from the Maven build
@@ -149,7 +147,7 @@ fi
149147

150148
if [[ "$1" =~ org.apache.spark.tools.* ]]; then
151149
if test -z "$SPARK_TOOLS_JAR"; then
152-
echo "Failed to find Spark Tools Jar in $FWDIR/tools/target/scala-$SCALA_VERSION/" 1>&2
150+
echo "Failed to find Spark Tools Jar in $FWDIR/tools/target/scala-$SPARK_SCALA_VERSION/" 1>&2
153151
echo "You need to build Spark before running $1." 1>&2
154152
exit 1
155153
fi

0 commit comments

Comments
 (0)